The ultimate purpose of MAAS is to deploy and manage machines. It’s important to understand how images get deployed – as explained in About the machine life-cycle, machines must first be enlisted or commissioned, then allocated, then deployed. This article will help you build on that understanding by explaining:
Note that if you are using your own commissioning scripts, and you do not want them to automatically run every time, you must specify noauto
, as in this script snippet:
#!/bin/bash
#
# --- Start MAAS 1.0 script metadata ---
# name: 50-script-example
# title: Example
# description: Just an example
# script_type: commissioning
# tags: noauto
If you do not specify noauto
, your custom commissioning scripts will run every time commissioning is attempted.
To commission a machine that’s in the “Ready” state, via the CLI, use the following command:
maas $PROFILE machine commission $SYSTEM_ID
Typical JSON output (long listing)
Success.
Machine-readable output follows:
{
"storage_test_status_name": "Pending",
"bcaches": [],
"cpu_count": 1,
"interface_set": [
{
"params": "",
"numa_node": 0,
"tags": [],
"id": 10,
"mac_address": "52:54:00:15:36:f2",
"vendor": "Red Hat, Inc.",
"children": [],
"effective_mtu": 1500,
"discovered": [],
"links": [],
"link_speed": 0,
"link_connected": true,
"system_id": "bhxws3",
"enabled": true,
"interface_speed": 0,
"firmware_version": null,
"name": "ens3",
"sriov_max_vf": 0,
"product": null,
"vlan": {
"vid": 0,
"mtu": 1500,
"dhcp_on": true,
"external_dhcp": null,
"relay_vlan": null,
"fabric": "fabric-2",
"primary_rack": "8dwnne",
"name": "untagged",
"id": 5003,
"space": "undefined",
"secondary_rack": null,
"fabric_id": 2,
"resource_uri": "/MAAS/api/2.0/vlans/5003/"
},
"parents": [],
"type": "physical",
"resource_uri": "/MAAS/api/2.0/nodes/bhxws3/interfaces/10/"
}
],
"network_test_status_name": "Unknown",
"numanode_set": [
{
"index": 0,
"memory": 985,
"cores": [
0
]
}
],
"locked": false,
"hardware_uuid": "F677A842-571C-4E65-ADC9-11E2CF92D363",
"default_gateways": {
"ipv4": {
"gateway_ip": null,
"link_id": null
},
"ipv6": {
"gateway_ip": null,
"link_id": null
}
},
"status_action": "",
"status_message": "Commissioning",
"cpu_test_status_name": "Unknown",
"memory_test_status": -1,
"virtualblockdevice_set": [],
"pool": {
"name": "default",
"description": "Default pool",
"id": 0,
"resource_uri": "/MAAS/api/2.0/resourcepool/0/"
},
"current_testing_result_id": 9,
"current_installation_result_id": null,
"netboot": true,
"description": "",
"special_filesystems": [],
"testing_status": 0,
"memory": 1024,
"current_commissioning_result_id": 8,
"storage": 5368.70912,
"commissioning_status": 0,
"cpu_test_status": -1,
"tag_names": [
"virtual"
],
"memory_test_status_name": "Unknown",
"swap_size": null,
"status_name": "Commissioning",
"other_test_status": -1,
"pod": null,
"storage_test_status": 0,
"blockdevice_set": [
{
"id_path": "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001",
"size": 5368709120,
"block_size": 512,
"tags": [
"ssd"
],
"serial": "QM00001",
"uuid": null,
"numa_node": 0,
"available_size": 5368709120,
"id": 3,
"partition_table_type": null,
"model": "QEMU HARDDISK",
"path": "/dev/disk/by-dname/sda",
"storage_pool": null,
"used_for": "Unused",
"filesystem": null,
"system_id": "bhxws3",
"used_size": 0,
"partitions": [],
"name": "sda",
"type": "physical",
"resource_uri": "/MAAS/api/2.0/nodes/bhxws3/blockdevices/3/"
}
],
"other_test_status_name": "Unknown",
"distro_series": "",
"testing_status_name": "Pending",
"ip_addresses": [],
"address_ttl": null,
"system_id": "bhxws3",
"physicalblockdevice_set": [
{
"firmware_version": "2.5+",
"serial": "QM00001",
"uuid": null,
"numa_node": 0,
"available_size": 5368709120,
"size": 5368709120,
"tags": [
"ssd"
],
"id": 3,
"partition_table_type": null,
"id_path": "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001",
"model": "QEMU HARDDISK",
"path": "/dev/disk/by-dname/sda",
"storage_pool": null,
"used_for": "Unused",
"filesystem": null,
"system_id": "bhxws3",
"used_size": 0,
"partitions": [],
"name": "sda",
"block_size": 512,
"type": "physical",
"resource_uri": "/MAAS/api/2.0/nodes/bhxws3/blockdevices/3/"
}
],
"fqdn": "ace-swan.maas",
"osystem": "",
"domain": {
"authoritative": true,
"ttl": null,
"resource_record_count": 0,
"name": "maas",
"id": 0,
"is_default": true,
"resource_uri": "/MAAS/api/2.0/domains/0/"
},
"boot_interface": {
"params": "",
"numa_node": 0,
"tags": [],
"id": 10,
"mac_address": "52:54:00:15:36:f2",
"vendor": "Red Hat, Inc.",
"children": [],
"effective_mtu": 1500,
"discovered": [],
"links": [],
"link_speed": 0,
"link_connected": true,
"system_id": "bhxws3",
"enabled": true,
"interface_speed": 0,
"firmware_version": null,
"name": "ens3",
"sriov_max_vf": 0,
"product": null,
"vlan": {
"vid": 0,
"mtu": 1500,
"dhcp_on": true,
"external_dhcp": null,
"relay_vlan": null,
"fabric": "fabric-2",
"primary_rack": "8dwnne",
"name": "untagged",
"id": 5003,
"space": "undefined",
"secondary_rack": null,
"fabric_id": 2,
"resource_uri": "/MAAS/api/2.0/vlans/5003/"
},
"parents": [],
"type": "physical",
"resource_uri": "/MAAS/api/2.0/nodes/bhxws3/interfaces/10/"
},
"hostname": "ace-swan",
"network_test_status": -1,
"min_hwe_kernel": "",
"power_state": "off",
"interface_test_status_name": "Unknown",
"owner_data": {},
"volume_groups": [],
"power_type": "virsh",
"node_type": 0,
"owner": "admin",
"cache_sets": [],
"architecture": "amd64/generic",
"hwe_kernel": null,
"zone": {
"name": "default",
"description": "",
"id": 1,
"resource_uri": "/MAAS/api/2.0/zones/default/"
},
"disable_ipv4": false,
"boot_disk": {
"firmware_version": "2.5+",
"serial": "QM00001",
"uuid": null,
"numa_node": 0,
"available_size": 5368709120,
"size": 5368709120,
"tags": [
"ssd"
],
"id": 3,
"partition_table_type": null,
"id_path": "/dev/disk/by-id/ata-QEMU_HARDDISK_QM00001",
"model": "QEMU HARDDISK",
"path": "/dev/disk/by-dname/sda",
"storage_pool": null,
"used_for": "Unused",
"filesystem": null,
"system_id": "bhxws3",
"used_size": 0,
"partitions": [],
"name": "sda",
"block_size": 512,
"type": "physical",
"resource_uri": "/MAAS/api/2.0/nodes/bhxws3/blockdevices/3/"
},
"status": 1,
"iscsiblockdevice_set": [],
"raids": [],
"node_type_name": "Machine",
"hardware_info": {
"system_vendor": "QEMU",
"system_product": "Standard PC (i440FX + PIIX, 1996)",
"system_family": "Unknown",
"system_version": "pc-i440fx-focal",
"system_sku": "Unknown",
"system_serial": "Unknown",
"cpu_model": "Intel Core Processor (Skylake, IBRS)",
"mainboard_vendor": "Unknown",
"mainboard_product": "Unknown",
"mainboard_serial": "Unknown",
"mainboard_version": "Unknown",
"mainboard_firmware_vendor": "SeaBIOS",
"mainboard_firmware_date": "04/01/2014",
"mainboard_firmware_version": "1.13.0-1ubuntu1",
"chassis_vendor": "QEMU",
"chassis_type": "Other",
"chassis_serial": "Unknown",
"chassis_version": "pc-i440fx-focal"
},
"commissioning_status_name": "Pending",
"bios_boot_method": "pxe",
"interface_test_status": -1,
"cpu_speed": 0,
"resource_uri": "/MAAS/api/2.0/machines/bhxws3/"
}
If you need to find the machine’s $SYSTEM_ID, you can use a command like this one:
maas $PROFILE machines read | jq '.[] | .hostname, .system_id'
"ace-swan"
"bhxws3"
Once commissioned, you may consider adding a tag to this machine. The next step is testing and deployment.
This section explains:
You can also refer to technical details and examples for commissioning scripts and testing scripts as needed.
How to download built-in scripts
You can download the source for all commissioning and test scripts via the API with the following command:
maas $PROFILE node-script download $SCRIPT_NAME
The source code to all built-in scripts is available on launchpad↗
.
How to upload hardware testing scripts
To upload a hardware testing script to MAAS, enter the following:
maas $PROFILE node-scripts create name=$SCRIPT_NAME name> \
script=$PATH_TO_SCRIPT type=testing
Changing the type to commissioning adds the test script to the commissioning process.
How to list all uploaded hardware testing scripts
You can list all uploaded scripts with the following command:
maas $PROFILE node-scripts read type=testing filters=$TAG
The optional filters argument lets you search for tags assigned to a script, such as using TAG=cpu
with the above example.
How to update hardware testing scripts
A script’s metadata, and even the script itself, can be updated from the command line:
maas $PROFILE node-script update \
$SCRIPT_NAME script=$PATH_TO_SCRIPT comment=$COMMENT
The JSON formatted output to the above command will include ‘history’ dictionary entries, detailing script modification times and associated comments:
"history": [
{
"id": 40,
"created": "Tue, 12 Sep 2017 12:12:08 -0000",
"comment": "Updated version"
},
{
"id": 34,
"created": "Fri, 08 Sep 2017 17:07:46 -0000",
"comment": null
}
]
How to revert hardware testing scripts
MAAS keeps a history of all uploaded script versions, allowing you to easily revert to a previous version, using the id
of the desired version:
maas $PROFILE node-script revert $SCRIPT_NAME to=$VERSION_ID
NOTE: The history for later modifications will be lost when reverting to an earlier version of the script.
To download a script, enter the following:
maas $PROFILE node-script download $SCRIPT_NAME > $LOCAL_FILENAME
To delete a script, use delete
:
maas $PROFILE node-script delete $SCRIPT_NAME
The command line allows you to not only view the current script’s progress but also retrieve the verbatim output from any previous runs too.
If you only want to see the latest or currently-running result, you can use current-commissioning
, current-testing
, or current-installation
instead of an id:
maas $PROFILE node-script-result read $SYSTEM_ID $RESULTS
You can also limit which results are returned by type (commissioning, testing, or installation), script name, or script run:
maas $PROFILE node-script-results read \
$SYSTEM_ID type=$SCRIPT_TYPE filters=$SCRIPT_NAME,$TAGS
How to suppress failed results
You can also suppress failed results, which is useful if you want to ignore a known failure:
maas $PROFILE node-script-results update \
$SYSTEM_ID type=$SCRIPT_TYPE filters=$SCRIPT_NAME,$TAGS suppressed=$SUPPRESSED
where $SUPPRESSED
is either True
or False
. The JSON formatted output to the above command will include ‘results’ dictionary with an entry for suppressed
:
"results": [
{
"id": 21,
"created": "Tue, 02 Apr 2019 17:00:36 -0000",
"updated": "Tue, 02 Apr 2019 20:56:41 -0000",
"name": "smartctl-validate",
"status": 5,
"status_name": "Aborted",
"exit_status": null,
"started": "Tue, 02 Apr 2019 20:56:41 -0000",
"ended": "Tue, 02 Apr 2019 20:56:41 -0000",
"runtime": "0:00:00",
"starttime": 1554238601.765214,
"endtime": 1554238601.765214,
"estimated_runtime": "0:00:00",
"parameters": {
"storage": {
"argument_format": "{path}",
"type": "storage",
"value": {
"id_path": "/dev/vda",
"model": "",
"name": "sda",
"physical_blockdevice_id": 1,
"serial": ""
}
}
},
"script_id": 1,
"script_revision_id": null,
"suppressed": true
}
]
Finally, results can be downloaded, either to stdout, stderr, as combined output or as a tar.xz:
maas $PROFILE node-script-result download $SYSTEM_ID $RUN_ID output=all \
filetype=tar.xz > $LOCAL_FILENAME
NOTE:
$RUN_ID is labelled id
in the verbose result output.
Commissioning and testing script files may be found in the following directories:
/tmp/user_data.sh.*/scripts/commissioning/
: Commissioning scripts/tmp/user_data.sh.*/scripts/testing/
: Hardware testing scriptsCommissioning and testing log files may be found in the following directories:
/tmp/user_data.sh*/out/
/var/log/cloud-init-output.log
/var/log/cloud-init.log
How to run all scripts manually
You can also run all commissioning and hardware-testing scripts on a machine for debugging.
/tmp/user_data.sh.*/bin/maas-run-remote-scripts \
[--no-download] \
[--no-send] \
/tmp/user_data.sh.*
Where:
--no-download
: Optional. Do not download the scripts from MAAS again.--no-send
: Optional. Do not send the results to MAAS.For example, to run all the scripts again without downloading again from MAAS:
/tmp/user_data.sh.*/bin/maas-run-remote-scripts --no-download /tmp/user_data.sh.*
Here, all the scripts are run again after downloading from MAAS, but no output data is sent to MAAS:
/tmp/user_data.sh.*/bin/maas-run-remote-scripts --no-send /tmp/user_data.sh.*
How to upload hardware test scripts
To upload a hardware testing script to MAAS, enter the following:
maas $PROFILE node-scripts create name=$SCRIPT_NAME name> \
script=$PATH_TO_SCRIPT type=testing
Changing the type to commissioning adds the test script to the commissioning process.
You can list all uploaded scripts with the following command:
maas $PROFILE node-scripts read type=testing filters=$TAG
The optional filters argument lets you search for tags assigned to a script, such as using TAG=cpu
with the above example.
A script’s metadata, and even the script itself, can be updated from the command line:
maas $PROFILE node-script update \
$SCRIPT_NAME script=$PATH_TO_SCRIPT comment=$COMMENT
The JSON formatted output to the above command will include ‘history’ dictionary entries, detailing script modification times and associated comments:
"history": [
{
"id": 40,
"created": "Tue, 12 Sep 2017 12:12:08 -0000",
"comment": "Updated version"
},
{
"id": 34,
"created": "Fri, 08 Sep 2017 17:07:46 -0000",
"comment": null
}
]
MAAS keeps a history of all uploaded script versions, allowing you to easily revert to a previous version using the id
of the version you wish to revert to:
maas $PROFILE node-script revert $SCRIPT_NAME to=$VERSION_ID
NOTE: The history for later modifications will be lost when reverting to an earlier version of the script.
To download a script, enter the following:
maas $PROFILE node-script download $SCRIPT_NAME > $LOCAL_FILENAME
To delete a script, use delete
:
maas $PROFILE node-script delete $SCRIPT_NAME
How to use tags to group commissioning and testing scripts
Tags make scripts easier to manage; grouping scripts together for commissioning and testing, for example:
maas $PROFILE node-script add-tag $SCRIPT_NAME tag=$TAG
maas $PROFILE node-script remove-tag $SCRIPT_NAME tag=$TAG
MAAS runs all commissioning scripts by default. However, you can select which custom scripts to run during commissioning by name or tag:
maas $PROFILE machine commission \
commissioning_scripts=$SCRIPT_NAME,$SCRIPT_TAG
You can also select which testing scripts to run by name or tag:
maas $PROFILE machine commission \
testing_scripts=$SCRIPT_NAME,$SCRIPT_TAG
Any testing scripts tagged with commissioning will also run during commissioning.
The command line allows you to not only view the current script’s progress but also retrieve the verbatim output from any previous runs too.
If you only want to see the latest or currently-running result, you can use current-commissioning
, current-testing
, or current-installation
instead of an id:
maas $PROFILE node-script-result read $SYSTEM_ID $RESULTS
You can also limit which results are returned by type (commissioning, testing, or installation), script name, or script run:
maas $PROFILE node-script-results read \
$SYSTEM_ID type=$SCRIPT_TYPE filters=$SCRIPT_NAME,$TAGS
You can also suppress failed results, which is useful if you want to ignore a known failure:
maas $PROFILE node-script-results update \
$SYSTEM_ID type=$SCRIPT_TYPE filters=$SCRIPT_NAME,$TAGS suppressed=$SUPPRESSED
where $SUPPRESSED
is either True
or False
. The JSON formatted output to the above command will include ‘results’ dictionary with an entry for suppressed
:
"results": [
{
"id": 21,
"created": "Tue, 02 Apr 2019 17:00:36 -0000",
"updated": "Tue, 02 Apr 2019 20:56:41 -0000",
"name": "smartctl-validate",
"status": 5,
"status_name": "Aborted",
"exit_status": null,
"started": "Tue, 02 Apr 2019 20:56:41 -0000",
"ended": "Tue, 02 Apr 2019 20:56:41 -0000",
"runtime": "0:00:00",
"starttime": 1554238601.765214,
"endtime": 1554238601.765214,
"estimated_runtime": "0:00:00",
"parameters": {
"storage": {
"argument_format": "{path}",
"type": "storage",
"value": {
"id_path": "/dev/vda",
"model": "",
"name": "sda",
"physical_blockdevice_id": 1,
"serial": ""
}
}
},
"script_id": 1,
"script_revision_id": null,
"suppressed": true
}
]
Finally, results can be downloaded, either to stdout, stderr, as combined output or as a tar.xz:
maas $PROFILE node-script-result download $SYSTEM_ID $RUN_ID output=all \
filetype=tar.xz > $LOCAL_FILENAME
NOTE:
$RUN_ID is labelled id
in the verbose result output.
MAAS can check whether links are connected or disconnected, so that you can detect unplugged cables. If you are not running MAAS 2.7 or higher, you must first upgrade and then recommission your machines to find disconnected links. MAAS not only reports unplugged cables, but also gives a warning when trying to configure a disconnected interface. In addition, administrators can change the cable connection status after manually resolving the issue.
To check network testing results, enter the following command:
maas $PROFILE interfaces read $SYSTEM_ID \
| jq -r '(["LINK_NAME","LINK_CONNECTED?","LINK_SPEED", "I/F_SPEED"]
| (., map(length*"-"))), (.[] | [.name, .link_connected, .link_speed, .interface_speed])
| @tsv' | column -t
which produces an output similar to this:
LINK_NAME LINK_CONNECTED? LINK_SPEED I/F_SPEED
--------- --------------- ---------- ---------
ens3 false - 1 Gpbs
From this screen, you can see that the ens3
link is not connected (hence an unreported link speed).
Once you have manually repaired the broken connection, an administrator can change cable connection status:
maas $PROFILE interface update $SYSTEM_ID $INTERFACE_ID link_connected=true
How to detect slow network links
As servers and hardware get faster, the chances increase that you might encounter a speed mismatch when connecting your NIC to a network device. MAAS can warn you if your interface is connected to a link slower than what the interface supports, when you run the above command:
maas $PROFILE interfaces read $SYSTEM_ID \
| jq -r '(["LINK_NAME","LINK_CONNECTED?","LINK_SPEED", "I/F_SPEED"]
| (., map(length*"-"))), (.[] | [.name, .link_connected, .link_speed, .interface_speed])
| @tsv' | column -t
From the resulting output, you can detect when your link/interface speeds are slower than expected. Depending on your physical hardware, the problem may not be repairable, but once you identify a slow link, you can replace a slow switch without recommissioning.
Administrators can change or update the link and interface speeds after manual changes to the connection:
maas $PROFILE interface update $SYSTEM_ID $INTERFACE_ID link_speed=$NEW_LINK_SPEED \
interface_speed=$NEW_INTERFACE_SPEED
This functionality is only available through the MAAS CLI.
How to configure network validation and testing scripts
MAAS allows you to configure network connectivity testing in a number of ways. If MAAS can’t connect to the rack controller, deployment can’t complete. MAAS can check connectivity to the rack controller and warn you if there’s no link, long before you have to try and debug it. For example, if you can’t connect to your gateway controller, traffic can’t leave your network.
Via the Web UI only, MAAS can check this link and recognise that there’s no connectivity, which alleviates hard-to-detect network issues:
Users can now test their network configuration to check for:
In addition, MAAS can comprehensively test Internet connectivity testing. You can give a list of URLs or IP addresses to check:
In the ephemeral environment, standard DHCP is still applied, but when network testing runs, MAAS can apply your specific configuration for the duration of the test. While all URLs / IPs are tested with all interfaces, MAAS can test each of your interfaces individually, including breaking apart bonded NICS and testing each side of your redundant interfaces. You can also run different tests on each pass, e.g., a different set of URLs, although each run would be a different testing cycle.
To test individual interfaces, for example, you could issue the following CLI command:
Note that in this command, we are testing internet connectivity to the single interface “br0.”
How to customise network testing
MAAS allow you to customise network testing according to your needs. You can create your own commissioning scripts and tests related to networking, and you can run them during the network testing portion of the MAAS workflow.
There are no particular restrictions on these scripts, so you can test a wide variety of possible conditions and situations. Administrators can upload network tests and test scripts. Administrators can also create tests which accept an interface parameter, or scripts which apply custom network configurations.
Users can specify unique parameters using the API, override machines which fail network testing (allowing their use), and suppress individual failed network tests. Users can also review the health status from all interface tests, even sorting them by interface name and MAC. In addition, MAAS can report the overall status of all interfaces.
To allocate a random node:
maas $PROFILE machines allocate
To allocate a specific node:
maas $PROFILE machines allocate system_id=$SYSTEM_ID
NOTE: To allocate a node, it must have a status of ‘Ready’.
To deploy a node:
maas $PROFILE machine deploy $SYSTEM_ID
To deploy a node as a KVM host:
maas $PROFILE machine deploy $SYSTEM_ID install_kvm=True
NOTE: To deploy with the CLI, the node must have a status of ‘Allocated’. See ‘Allocate a node’ above.
By default, when you deploy a machine, MAAS will consider the deployment a failure if it doesn’t complete within 30 minutes. You can configure this timeout, if you wish, with the command:
maas $PROFILE maas set-config name=node-timeout value=$NUMBER_OF_MINUTES
How to enlist a machine that’s already running a workload
In order to add machine that’s already running a workload, there are currently two options:
Via the API/CLI, you can create a machine, passing the deployed flag:
$ maas $profile machines create deployed=true hostname=mymachine \
architecture=amd64 mac_addresses=00:16:3e:df:35:bb power_type=manual
On the machine itself (the recommended way, if the machine is running Ubuntu), you can download a helper script from MAAS and create the machine that way:
$ wget http://$MAAS_IP:5240/MAAS/maas-run-scripts
$ chmod 755 maas-run-scripts
$ ./maas-run-scripts register-machine --hostname mymachine \
> http://$MAAS_IP:5240/MAAS $MAAS_API_TOKEN
Now you have a machine in MAAS that’s in the deployed state, with no hardware information yet.
How to update hardware information for a deployed machine
The recommended way of updating the hardware information for a deployed machine is to download the maas-run-scripts script and run it on the machine itself:
$ wget http://$MAAS_IP:5240/MAAS/maas-run-scripts
$ chmod 755 maas-run-scripts
$ ./maas-run-scripts report-results --config mymachine-creds.yaml
If you created the machine with the maas-run-scripts, you should have such a mymachine-creds.yaml file already. If not, it should look like this:
reporting:
maas:
consumer_key: $CONSUMER_KEY
endpoint: http://$MAAS_IP:5240/MAAS/metadata/status/$SYSTEM_ID
token_key: $TOKEN_KEY
token_secret: $TOKEN_SECRET
You may get the needed credentials from the MAAS API, for example:
$ maas $profile machine get-token wxwwga
Success.
Machine-readable output follows:
{
"token_key": "Lyy9BS4tKsQakDQScy",
"token_secret": "V8vta8Azwn6FZVkfHnuTvLGLScAvEufB",
"consumer_key": "YGT6QKSH65aap4tGnw"
}