...
 
Commits (14)
stages:
- test
- analyze
- build
- publish
- deploy
image: debian:latest
###########################
# Templates #
# #
# make sure all jobs use #
# architecture tags. #
###########################
.job: &job
before_script:
- ./.gitlab-ci/$CI_JOB_STAGE/$CI_JOB_NAME.sh setup
dependencies: []
tags:
- amd64
script:
- ./.gitlab-ci/$CI_JOB_STAGE/$CI_JOB_NAME.sh
.job_aarch64: &job_aarch64
<<: *job
tags:
- aarch64
.dind: &dind
<<: *job
tags:
- amd64
- dind
image: docker:stable
variables:
DOCKER_DRIVER: overlay2
services:
- docker:stable-dind
#######
# Tests
#######
.dispatcher: &dispatcher
<<: *job
stage: test
image: hub.lavasoftware.org/lava/ci-images/amd64/$CI_JOB_NAME
artifacts:
reports:
junit: dispatcher.xml
.server: &server
<<: *job
stage: test
image: hub.lavasoftware.org/lava/ci-images/amd64/$CI_JOB_NAME
artifacts:
reports:
junit:
- common.xml
- server.xml
dispatcher-debian-9: *dispatcher
server-debian-9: *server
dispatcher-debian-10: *dispatcher
server-debian-10: *server
###########
# Analyze #
###########
black:
<<: *job
stage: analyze
image: hub.lavasoftware.org/lava/ci-images/amd64/codestyle
before_script:
- apt-get update
- apt-get install -y --no-install-recommends devscripts dpkg-dev
- apt-get install -y --no-install-recommends python3-sphinx python3-setuptools python3-yaml python3-sphinx-bootstrap-theme node-uglify libjs-excanvas libjs-jquery-cookie libjs-jquery libjs-jquery-watermark libjs-jquery-flot libjs-jquery-typeahead libjs-jquery-ui build-essential debhelper po-debconf git
codestyle:
<<: *job
stage: analyze
image: hub.lavasoftware.org/lava/ci-images/amd64/codestyle
coverage:
<<: *job
stage: analyze
image: hub.lavasoftware.org/lava/ci-images/amd64/server-debian-9
coverage: '/^TOTAL.+ (\d+\.\d+)%$/'
artifacts:
paths:
- htmlcov
only:
refs:
- master
- tags
sast:
<<: *dind
stage: analyze
allow_failure: true
artifacts:
paths: [gl-sast-report.json]
#########
# Build #
#########
doc:
<<: *job
stage: build
image: debian:9
artifacts:
paths:
- doc/v2/_build/html
amd64/pkg-debian-9:
<<: *job
stage: build
image: hub.lavasoftware.org/lava/ci-images/amd64/pkg-debian-9
artifacts:
paths:
- _build/*.deb
# handle native package - need to publish the source
- _build/*.tar.xz
- _build/*.dsc
- _build/*.changes
- _build/*.buildinfo
amd64/pkg-debian-10:
<<: *job
stage: build
image: hub.lavasoftware.org/lava/ci-images/amd64/pkg-debian-10
artifacts:
paths:
- _build/*.deb
# handle native package - need to publish the source
- _build/*.tar.xz
- _build/*.dsc
- _build/*.changes
- _build/*.buildinfo
aarch64/pkg-debian-10:
<<: *job_aarch64
stage: build
image: hub.lavasoftware.org/lava/ci-images/aarch64/pkg-debian-10
only:
refs:
- master
- tags
artifacts:
# binary only build, no .dsc or .tar.xz
paths:
- _build/*.deb
- _build/*.changes
- _build/*.buildinfo
aarch64/pkg-debian-9:
<<: *job_aarch64
stage: build
image: hub.lavasoftware.org/lava/ci-images/aarch64/pkg-debian-9
only:
refs:
- master
- tags
build:
script:
- dpkg-buildpackage -uc -b
- mkdir debs
- mv ../*deb debs/
artifacts:
# binary only build, no .dsc or .tar.xz
paths:
- _build/*.deb
- _build/*.changes
- _build/*.buildinfo
###########
# Publish #
###########
.publish: &publish
<<: *dind
stage: publish
dependencies: [amd64/pkg-debian-9]
variables:
IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_JOB_NAME
.publish_aarch64: &publish_aarch64
<<: *publish
only:
refs:
- master
- tags
tags: [dind, aarch64]
dependencies: [aarch64/pkg-debian-10, amd64/pkg-debian-9]
amd64/lava-dispatcher: *publish
amd64/lava-server: *publish
aarch64/lava-dispatcher: *publish_aarch64
aarch64/lava-server: *publish_aarch64
##########
# Deploy #
##########
lavafed-master:
<<: *job
stage: deploy
tags: [deploy]
only: [schedules]
environment:
name: lavafed-master
url: https://federation.lavasoftware.org/lava/
- 'debs/*.deb'
expire_in: 1 week
reprepro-master:
<<: *job
stage: deploy
# Environments and therefore runners are tied to
# a specific branch. You'll see this message:
# "This job is creating a deployment to reprepro-master and
# will overwrite the latest deployment."
tags: [deploy]
dependencies:
- aarch64/pkg-debian-9
- aarch64/pkg-debian-10
- amd64/pkg-debian-9
- amd64/pkg-debian-10
only: [schedules]
environment:
name: reprepro-master
url: https://apt.lavasoftware.org
lava (2018.11-2) unstable; urgency=medium
* Non-maintainer upload.
* improve logging for Invalid device configurations
* remove validation check which forbids ssh and power_off/on commands
-- Alexander Couzens <lynxis@fe80.eu> Sun, 02 Dec 2018 05:41:18 +0000
lava (2018.11-1) unstable; urgency=medium
* 2018.11-1 LAVA Software 2018.11 release
......
......@@ -5,7 +5,7 @@ Maintainer: Debian LAVA team <pkg-linaro-lava-devel@lists.alioth.debian.org>
Uploaders: Neil Williams <codehelp@debian.org>,
Senthil Kumaran S (stylesen) <stylesen@gmail.com>
Build-Depends:
debhelper (>= 9.20160709), dh-python, po-debconf,
debhelper (>= 9.20160709), dh-python, po-debconf, git,
python3, python3-sphinx (>= 1.4), python3-setuptools (>= 3),
python3-yaml, python3-sphinx-bootstrap-theme,
node-uglify, libjs-excanvas, libjs-jquery-cookie, libjs-jquery,
......
......@@ -425,7 +425,7 @@ class Action: # pylint: disable=too-many-instance-attributes,too-many-public-me
def get_constant(self, key, prefix):
# whilst deployment data is still supported, check if the key exists there.
# once deployment_data is removed, merge with device.get_constant
if self.parameters.get('deployment_data'):
if self.parameters.get('deployment_data') and key in self.parameters['deployment_data']:
return self.parameters['deployment_data'][key]
return self.job.device.get_constant(key, prefix=prefix)
......
......@@ -360,6 +360,7 @@ class AutoLoginAction(Action):
self.logger.debug("Running login commands")
for command in login_commands:
connection.sendline(command)
self.wait(connection)
return connection
......
......@@ -43,7 +43,7 @@ class DeployDeviceEnvironment(Action):
super().validate()
shell_file = self.get_constant('lava_test_shell_file', 'posix')
if not shell_file:
self.errors = "Invalid deployment data - missing lava_test_shell_file"
self.logger.error("Missing lava_test_shell_file. Can not setup environment file.")
if 'env_dut' in self.job.parameters and self.job.parameters['env_dut']:
# Check that the file is valid yaml
......
......@@ -249,7 +249,7 @@ if list(Device.objects.filter(Q(device_type=dt), ~Q(health=Device.HEALTH_RETIRED
return device_types
def load_devicetype_template(device_type_name, raw=False):
def load_devicetype_template(device_type_name, raw=False, exception=False):
"""
Loads the bare device-type template as a python dictionary object for
representation within the device_type templates.
......@@ -272,7 +272,10 @@ def load_devicetype_template(device_type_name, raw=False):
return None
return data if raw else yaml.safe_load(data)
except (jinja2.TemplateError, yaml.error.YAMLError):
return None
if exception:
raise
else:
return None
def invalid_template(dt):
......@@ -306,6 +309,12 @@ def invalid_template(dt):
d_template = False # template exists, invalid check is False
return d_template
def invalid_template_error(dt):
try:
d_template = load_devicetype_template(dt.name)
except Exception as excep:
return str(excep)
return None
def validate_job(data):
try:
......
......@@ -672,6 +672,14 @@ class Device(RestrictedResource):
return False
return True
def get_validate_error(self):
try:
rendered = self.load_configuration()
validate_device(rendered)
except (SubmissionException, yaml.YAMLError) as exp:
return str(exp)
return ""
def log_admin_entry(self, user, reason):
if user is None:
user = User.objects.get(username="lava-health")
......
......@@ -248,10 +248,14 @@ def schedule_jobs_for_device(logger, device):
% (prev_health_display, device.get_health_display()),
)
device.save()
logger.debug(
logger.error(
"%s → %s (Invalid device configuration for %s)"
% (prev_health_display, device.get_health_display(), device.hostname)
)
logger.error(
"%s: %s"
% (device.hostname, device.get_validate_error())
)
continue
job_dict = yaml.safe_load(job.definition)
......
......@@ -479,22 +479,6 @@ def validate_submission(data_object):
return True
def _validate_primary_connection_power_commands(data_object):
power_control_commands = ["power_off", "power_on", "hard_reset"]
# debug, tests don't pass. write docs.
with contextlib.suppress(KeyError):
ssh_host = data_object["actions"]["deploy"]["methods"]["ssh"]["host"]
if ssh_host:
if "commands" in data_object:
for command in power_control_commands:
if command in data_object["commands"]:
raise SubmissionException(
"When primary connection is used, power control commands (%s) should not be specified."
% ", ".join(power_control_commands)
)
def validate_device(data_object):
"""
Validates a python object as a pipeline device configuration
......@@ -509,5 +493,4 @@ def validate_device(data_object):
except MultipleInvalid as exc:
raise SubmissionException(exc)
_validate_primary_connection_power_commands(data_object)
return True
......@@ -9,6 +9,7 @@
<strong>Configuration Error: missing or invalid template.</strong>
<p>Jobs requesting this device type ({{ device.device_type }}) will not be able to start until
a template is available on the master.</p>
{% if validate_errors is defined %}<p>{{ validate_errors }}</p>{% endif %}
</div>
{% endif %}
......
......@@ -8,6 +8,7 @@
<strong>Configuration Error: missing or invalid template.</strong>
<p>Jobs requesting this device type will not be able to start until
a template is available on the master.</p>
{% if invalid_template_error %}<p>{{ invalid_template_error }}</p>{% endif %}
</div>
{% endif %}
......
......@@ -75,6 +75,7 @@ from lava_scheduler_app.dbutils import (
load_devicetype_template,
testjob_submission,
validate_job,
invalid_template_error,
)
from lava_scheduler_app.logutils import read_logs
from lava_scheduler_app.templatetags.utils import udecode
......@@ -963,6 +964,7 @@ def device_type_detail(request, pk):
"context_help": BreadCrumbTrail.leading_to(device_type_detail, pk="help"),
"health_freq": health_freq_str,
"invalid_template": invalid_template(dt),
"invalid_template_error": invalid_template_error(dt),
},
)
......@@ -2000,6 +2002,7 @@ def device_detail(request, pk):
"previous_device": previous_device,
"overrides": overrides,
"template_mismatch": mismatch,
"validate_errors": device.get_validate_error(),
},
request=request,
)
......
......@@ -28,7 +28,7 @@ import sys
# pylint: disable=superfluous-parens,too-many-locals
def version_tag():
def version_tag(version=None):
"""
Parses the git status to determine if this is a git tag
or a developer commit and builds a version string combining
......@@ -39,9 +39,8 @@ def version_tag():
:return: a version string based on the tag and short hash
"""
args = ["git", "describe"]
if len(sys.argv) == 2:
if sys.argv[1] != 'sdist':
args.append(sys.argv[1])
if version:
args.append(version)
if os.path.exists("./.git/"):
pattern = re.compile(r"(?P<tag>.+)\.(?P<commits>\d+)\.g(?P<hash>[abcdef\d]+)")
describe = (
......@@ -62,7 +61,11 @@ def version_tag():
def main():
print(version_tag())
version = None
if len(sys.argv) == 2:
if sys.argv[1] != 'sdist':
version = sys.argv[1]
print(version_tag(version))
return 0
......