summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSenko Rasic <senko.rasic@dobarkod.hr>2011-08-19 12:46:58 (GMT)
committerSenko Rasic <senko.rasic@dobarkod.hr>2011-08-19 12:48:13 (GMT)
commita8edca29797a56884cde9e7f7d464b41a3f3f750 (patch)
tree6a5dc2cb1caa7e9f38a7954d960d8ade27cc6eb3
parent4b154dece781a6f66f3f7c9f8fc91c604bb70da5 (diff)
downloadgst-qa-system-a8edca29797a56884cde9e7f7d464b41a3f3f750.tar.gz
gst-qa-system-a8edca29797a56884cde9e7f7d464b41a3f3f750.tar.xz
add expected failure support
Extra test params can specify under which situations a test item may fail.
-rw-r--r--insanity/test.py60
-rw-r--r--web/insanityweb/models.py38
-rw-r--r--web/insanityweb/runner.py9
-rw-r--r--web/insanityweb/views.py7
-rw-r--r--web/templates/insanityweb/current.html2
-rw-r--r--web/templates/insanityweb/matrix_checklist_row.html16
-rw-r--r--web/templates/insanityweb/test_checklist_dict.html6
7 files changed, 112 insertions, 26 deletions
diff --git a/insanity/test.py b/insanity/test.py
index a18511d..61de1a0 100644
--- a/insanity/test.py
+++ b/insanity/test.py
@@ -52,6 +52,11 @@ class Test(gobject.GObject):
@ivar uuid: unique identifier for the test
"""
+ SKIPPED = None
+ FAILURE = 0
+ SUCCESS = 1
+ EXPECTED_FAILURE = 2
+
__test_name__ = "test-base-class"
"""
Name of the test
@@ -220,6 +225,16 @@ class Test(gobject.GObject):
self._monitors = []
self._monitorinstances = []
+ # list of (checkitem, pattern) where pattern is either:
+ # - a function taking two params: checkitem, and test extra_info dict
+ # - a dict containing key/value pairs that must be present in test args
+ # - True for unconditional match
+ self._expected_failure_patterns = kwargs.get('expected_failures', [])
+
+ # dict containing (checkitem, True) for every expected failure that happened
+ self._expected_failures = {}
+
+
@classmethod
def get_file(cls):
"""
@@ -418,9 +433,33 @@ class Test(gobject.GObject):
if checkitem in dict(self._checklist):
return
self._checklist.append((checkitem, bool(validated)))
+
+ if not validated:
+ if self.isExpectedFailure(checkitem, self._extrainfo):
+ self._expected_failures[checkitem] = True
+ explanation = self.processFailure(checkitem, self._extrainfo)
+ if explanation is not None:
+ self._error_explanations[checkitem] = explanation
+
#self._checklist[checkitem] = True
self.emit("check", checkitem, validated)
+ def isExpectedFailure(self, checkitem, extra_info):
+ for pattern in self._expected_failure_patterns:
+ expected = True
+ for k, v in pattern.items():
+ if k == 'checkitem':
+ if v != checkitem:
+ expected = False
+ break
+ elif self.arguments.get(k, None) != v:
+ expected = False
+ break
+
+ if expected:
+ return True
+ return False
+
def extraInfo(self, key, value):
"""
Give extra information obtained while running the tests.
@@ -503,15 +542,24 @@ class Test(gobject.GObject):
Returns the instance checklist as a list of tuples of:
* checkitem name
* value indicating whether the success of that step
- That value can be one of : True, False, None
- None means that step was neither validated or invalidated.
+ That value can be one of: SKIPPED, SUCCESS, FAILURE, EXPECTED_FAILURE
"""
allk = self.getFullCheckList().keys()
- d = dict([(k,v) for k,v in self._checklist])
+
+ def to_enum(key, val):
+ if val:
+ return self.SUCCESS
+ elif self._expected_failures.get(key, False):
+ return self.EXPECTED_FAILURE
+ else:
+ return self.FAILURE
+
+ d = dict((k, to_enum(k, v)) for k, v in self._checklist)
for k in allk:
- if not d.has_key(k):
- d[k] = None
- return [(k,v) for k,v in d.iteritems()]
+ if k not in d:
+ d[k] = self.SKIPPED
+
+ return d.items()
def getArguments(self):
"""
diff --git a/web/insanityweb/models.py b/web/insanityweb/models.py
index 6e3a679..7f0d7d2 100644
--- a/web/insanityweb/models.py
+++ b/web/insanityweb/models.py
@@ -352,12 +352,12 @@ class Test(models.Model):
'type': TestClassInfoCheckListDict
'value': TestCheckListList
'skipped': boolean set to True if check was skipped
+ 'expected': boolean set to True if the check was expected failure
This differs from checklist_set in the sense that it will indicate
the skipped check items.
"""
res = []
-
# pre-computed fullchecklist
if checklist == None:
fcl = self.type.fullchecklist
@@ -373,14 +373,17 @@ class Test(models.Model):
for checktype in fcl:
d = {}
d['type'] = checktype
+ d['expected'] = False
val = None
for av in v:
if checktype == av.name:
d['skipped'] = False
- val = av.value
+ d['expected'] = av.expected
+ val = av.success
break
if val == None:
d['skipped'] = True
+
d['value'] = val
res.append(d)
return res
@@ -516,17 +519,44 @@ class TestArgumentsDict(models.Model):
def __str__(self):
return "%s:%s" % (self.name.name, self.value)
+
class TestCheckListList(models.Model):
+ SKIPPED = None
+ FAILURE = 0
+ SUCCESS = 1
+ EXPECTED_FAILURE = 2
+
+ VALUE_CHOICES = (
+ (SKIPPED, 'Skipped'),
+ (FAILURE, 'Failure'),
+ (SUCCESS, 'Success'),
+ (EXPECTED_FAILURE, 'Expected failure'),
+ )
+
id = models.IntegerField(null=True, primary_key=True, blank=True)
containerid = models.ForeignKey(Test, db_column="containerid",
related_name="checklist")
name = models.ForeignKey(TestClassInfoCheckListDict,
db_column="name")
value = models.IntegerField(null=True, blank=True,
- db_column="intvalue")
+ db_column="intvalue",
+ choices=VALUE_CHOICES)
+
@property
def skipped(self):
- return self.value == None
+ return self.value == self.SKIPPED
+
+ @property
+ def failure(self):
+ return self.value == self.FAILURE
+
+ @property
+ def success(self):
+ return self.value == self.SUCCESS
+
+ @property
+ def expected_failure(self):
+ return self.value == self.EXPECTED_FAILURE
class Meta:
db_table = 'test_checklist_list'
diff --git a/web/insanityweb/runner.py b/web/insanityweb/runner.py
index b3d6a44..d24d703 100644
--- a/web/insanityweb/runner.py
+++ b/web/insanityweb/runner.py
@@ -96,12 +96,15 @@ class Runner(object):
tests.extend(t[0] for t in insanity.utils.list_available_scenarios())
return tests
- def start_test(self, test, folder):
+ def start_test(self, test, folder, extra_arguments):
self.run = TestRun(maxnbtests=1)
self.test_class = insanity.utils.get_test_class(test)
- self.run.addTest(self.test_class, arguments={
+ args = {
'uri': URIFileSystemGenerator(paths=[folder], recursive=True)
- })
+ }
+ args.update(extra_arguments)
+
+ self.run.addTest(self.test_class, arguments=args)
self.client.addTestRun(self.run)
storage = SQLiteStorage(path=settings.DATABASES['default']['NAME'])
diff --git a/web/insanityweb/views.py b/web/insanityweb/views.py
index 8582c66..ebac4da 100644
--- a/web/insanityweb/views.py
+++ b/web/insanityweb/views.py
@@ -174,14 +174,15 @@ def current(request):
if 'submit' in request.POST:
test = request.POST.get('test', '')
folder = request.POST.get('folder', '')
- if test in test_names and folder in settings.INSANITY_TEST_FOLDERS:
- runner.start_test(test, folder)
+ if test in test and folder in settings.INSANITY_TEST_FOLDERS:
+ runner.start_test(test, folder,
+ settings.INSANITY_TEST_FOLDERS[folder].get('extra-arguments', {}))
return redirect('web.insanityweb.views.current')
progress = runner.get_progress()
tests_running = (progress is not None)
test = runner.get_test_name()
- folder = settings.INSANITY_TEST_FOLDERS.get(runner.get_test_folder(), '(unknown folder)')
+ folder = settings.INSANITY_TEST_FOLDERS.get(runner.get_test_folder(), {'name':'(unknown folder)'})['name']
return render_to_response("insanityweb/current.html", locals())
def stop_current(request):
diff --git a/web/templates/insanityweb/current.html b/web/templates/insanityweb/current.html
index 6f7fa02..ad8ce00 100644
--- a/web/templates/insanityweb/current.html
+++ b/web/templates/insanityweb/current.html
@@ -27,7 +27,7 @@ Insanity QA system - Start / Stop Test
in
<select name="folder">
{% for path, desc in test_folders %}
- <option value="{{ path }}">{{ desc }}</option>
+ <option value="{{ path }}">{{ desc.name }}</option>
{% endfor %}
</select>
<input type="submit" class="button" name="submit" value="Start Test Run" />
diff --git a/web/templates/insanityweb/matrix_checklist_row.html b/web/templates/insanityweb/matrix_checklist_row.html
index 56b9a89..75889c6 100644
--- a/web/templates/insanityweb/matrix_checklist_row.html
+++ b/web/templates/insanityweb/matrix_checklist_row.html
@@ -11,13 +11,17 @@
<td class="{% if test.is_success %}result-OK{% else %}result-FAIL{% endif %}">{{test.resultpercentage|floatformat:1}}%</td>
{% for item in results %}
{% if item.skipped %}
- <td class="result-skipped">
- <span title="{{item.name}} : skipped/unavailable">?</span>
- </td>
+ <td class="result-skipped"><span title="{{item.name}} : skipped/unavailable">?</span></td>
{% else %}
- <td class="{%if item.value %}result-OK{% else %}result-FAIL{% endif %}">
- <span title="{{item.name}} : {%if item.value %}OK{% else %}failed{% endif %}">{{ item.value }}</span>
- </td>
+ {% if item.success %}
+ <td class="result-OK"><span title="{{ item.name}} : OK">+</span></td>
+ {% else %}
+ {% if item.expected_failure %}
+ <td class="result-OK"><span title="{{ item.name}} : failed (expected)">~</span></td>
+ {% else %}
+ <td class="result-FAIL"><span title="{{ item.name}} : failed">-</span></td>
+ {% endif %}
+ {% endif %}
{% endif %}
{% endfor %}
diff --git a/web/templates/insanityweb/test_checklist_dict.html b/web/templates/insanityweb/test_checklist_dict.html
index 4ababb2..8cce35f 100644
--- a/web/templates/insanityweb/test_checklist_dict.html
+++ b/web/templates/insanityweb/test_checklist_dict.html
@@ -9,11 +9,11 @@
<th class="side">{{result.name}}</th>
{% if result.skipped %}
<td class="result-skipped">
- ?
+ Skip
</td>
{% else %}
- <td class="{%if result.value %}result-OK{% else %}result-FAIL{% endif %}">
- {{result.value}}
+ <td class="{%if result.success %}result-OK{% else %}{% if result.expected_failure %}result-OK{% else %}result-FAIL{% endif %}{% endif %}">
+ {% if result.success %}OK{% else %}Fail{% if result.expected_failure %} (expected){% endif %}{% endif %}
</td>
{% endif %}
<td>{{result.name.description}}</td>