summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSenko Rasic <senko.rasic@dobarkod.hr>2011-08-19 12:49:58 (GMT)
committerSenko Rasic <senko.rasic@dobarkod.hr>2011-08-19 12:50:27 (GMT)
commit3ec1b89df3cfc54b03dd64e1c1a1944c45dee4fd (patch)
tree51e9d6de05bc0b16a792e633b962a7f9de73943d
parent4ad9eff2d72d757f2877d8fcfe6b59e20ea71495 (diff)
downloadgst-qa-system-3ec1b89df3cfc54b03dd64e1c1a1944c45dee4fd.tar.gz
gst-qa-system-3ec1b89df3cfc54b03dd64e1c1a1944c45dee4fd.tar.xz
add failure explanation / diagnosis support
-rw-r--r--insanity/storage/dbstorage.py18
-rw-r--r--insanity/storage/sqlite.py7
-rw-r--r--insanity/test.py35
-rw-r--r--tests/typefind.py4
-rw-r--r--web/insanityweb/models.py14
-rw-r--r--web/insanityweb/templatetags/insanity_extra.py11
-rw-r--r--web/insanityweb/views.py18
-rw-r--r--web/templates/insanityweb/matrix_view.html20
-rw-r--r--web/templates/insanityweb/test_checklist_dict.html2
9 files changed, 126 insertions, 3 deletions
diff --git a/insanity/storage/dbstorage.py b/insanity/storage/dbstorage.py
index e33018a..56aa07f 100644
--- a/insanity/storage/dbstorage.py
+++ b/insanity/storage/dbstorage.py
@@ -320,6 +320,17 @@ class DBStorage(DataStorage, AsyncStorage):
return (testrunid, ttype, args, results, resperc,
extras, ofs, parentid, ismonitor, isscenario)
+ def getTestErrorExplanations(self, testid):
+ """
+ Returns a dict with explanations of failed test check
+ items. Only the failures that have explanations are
+ present in the dict.
+
+ * key : check item name
+ * value : explanation string
+ """
+ return self.__getDict("test_error_explanation_dict", testid, txtonly=True)
+
def getTestClassInfoFull(self, testtype, withparents=True):
searchstr = """SELECT id,parent,description,fulldescription
FROM testclassinfo WHERE type=?"""
@@ -982,6 +993,8 @@ class DBStorage(DataStorage, AsyncStorage):
test.__test_name__)
self.__storeTestOutputFileDict(tid, test.getOutputFiles(),
test.__test_name__)
+ self.__storeTestErrorExplanationDict(tid, test.getErrorExplanations(),
+ test.__test_name__)
# finally update the test
updatestr = "UPDATE test SET resultpercentage=?, parentid=? WHERE id=?"
@@ -1231,6 +1244,11 @@ class DBStorage(DataStorage, AsyncStorage):
return self.__storeDict("test_outputfiles_dict",
testid, map_dict(dic, maps))
+ def __storeTestErrorExplanationDict(self, testid, dic, testtype):
+ maps = self.__getTestClassCheckListMapping(testtype)
+ return self.__storeDict("test_error_explanation_dict",
+ testid, map_dict(dic, maps))
+
def __storeTestClassArgumentsDict(self, testclass, dic):
return self.__storeDict("testclassinfo_arguments_dict",
testclass, dic)
diff --git a/insanity/storage/sqlite.py b/insanity/storage/sqlite.py
index 7d861e7..4dbbc9e 100644
--- a/insanity/storage/sqlite.py
+++ b/insanity/storage/sqlite.py
@@ -180,6 +180,13 @@ CREATE TABLE test_extrainfo_dict (
txtvalue TEXT
);
+CREATE TABLE test_error_explanation_dict (
+ id INTEGER PRIMARY KEY,
+ containerid INTEGER,
+ name INTEGER,
+ txtvalue TEXT
+);
+
CREATE TABLE test_outputfiles_dict (
id INTEGER PRIMARY KEY,
containerid INTEGER,
diff --git a/insanity/test.py b/insanity/test.py
index 61de1a0..2e4dfbf 100644
--- a/insanity/test.py
+++ b/insanity/test.py
@@ -105,6 +105,15 @@ class Test(gobject.GObject):
* short description of the check item
"""
+ __test_likely_errors__ = {}
+ """
+ Dictionary of likely error causes for check items for which
+ a likely error cause can be assumed.
+
+ key : name of the check item
+ value : short description of the likely error cause
+ """
+
__test_timeout__ = 15
"""
Allowed duration for the test to run (in seconds).
@@ -234,6 +243,10 @@ class Test(gobject.GObject):
# dict containing (checkitem, True) for every expected failure that happened
self._expected_failures = {}
+ # dict containing (checkitem, explanation) for every failure that has
+ # an explanation
+ self._error_explanations = {}
+
@classmethod
def get_file(cls):
@@ -593,6 +606,17 @@ class Test(gobject.GObject):
"""
return self._outputfiles
+ def getErrorExplanations(self):
+ """
+ Returns dict of error explanations for failed check items. Only
+ the failed items that have some explanation are present in the
+ dict.
+
+ key : name of the failed check item
+ value : error explanation
+ """
+ return self._error_explanations
+
def getTimeout(self):
"""
Returns the currently configured timeout
@@ -646,7 +670,18 @@ class Test(gobject.GObject):
return False
self._monitors.append((monitor, monitorargs))
+ def processFailure(self, checkitem, extra_info):
+ """
+ Process the failure and return a human-readable description
+ of why it happened (string).
+
+ By default, returns the likely error explanation from
+ __test_checklist_errors__. Test subclasses can override
+ it to provide detailed processing.
+ Returns None if no explanation is available.
+ """
+ return self.__test_likely_errors__.get(checkitem, None)
# For compatibility:
diff --git a/tests/typefind.py b/tests/typefind.py
index f63d8cc..8b95f56 100644
--- a/tests/typefind.py
+++ b/tests/typefind.py
@@ -75,6 +75,10 @@ class TypeFindTest(GStreamerTest):
"streams" : "List of stream information (padname: length(ms), caps)"
}
+ __test_likely_errors__ = {
+ "is-media-type": "file is not a media file"
+ }
+
__pipeline_initial_state__ = gst.STATE_PAUSED
@classmethod
diff --git a/web/insanityweb/models.py b/web/insanityweb/models.py
index 7f0d7d2..8ba8ced 100644
--- a/web/insanityweb/models.py
+++ b/web/insanityweb/models.py
@@ -600,6 +600,20 @@ class TestExtraInfoDict(models.Model):
class Meta:
db_table = 'test_extrainfo_dict'
+class TestErrorExplanationDict(models.Model):
+ id = models.IntegerField(null=True, primary_key=True, blank=True)
+ containerid = models.ForeignKey(Test, db_column="containerid",
+ related_name="error_explanations")
+ name = models.ForeignKey(TestClassInfoCheckListDict,
+ db_column="name")
+ txtvalue = models.TextField(blank=True)
+
+ def __str__(self):
+ return "%s:%s" % (self.name.name, self.txtvalue)
+
+ class Meta:
+ db_table = 'test_error_explanation_dict'
+
class TestRunEnvironmentDict(models.Model):
id = models.IntegerField(null=True, primary_key=True, blank=True)
containerid = models.ForeignKey(TestRun, db_column="containerid",
diff --git a/web/insanityweb/templatetags/insanity_extra.py b/web/insanityweb/templatetags/insanity_extra.py
index ebeff67..1a297d5 100644
--- a/web/insanityweb/templatetags/insanity_extra.py
+++ b/web/insanityweb/templatetags/insanity_extra.py
@@ -157,7 +157,16 @@ def test_args_dict(test, fullarguments=None):
@register.inclusion_tag('insanityweb/test_checklist_dict.html')
def test_checklist_dict(test, fullchecklist=None):
- return {'results':test.checklist.all().select_related(depth=1)}
+ explanations = {}
+ for e in test.error_explanations.all().select_related(depth=1):
+ explanations[e.name.id] = e.txtvalue
+
+ results = []
+ for r in test.checklist.all().select_related(depth=1):
+ r.explanation = explanations.get(r.name.id, None)
+ results.append(r)
+
+ return {'results': results}
@register.inclusion_tag('insanityweb/test_extrainfo_dict.html')
def test_extrainfo_dict(test):
diff --git a/web/insanityweb/views.py b/web/insanityweb/views.py
index ebac4da..416b82a 100644
--- a/web/insanityweb/views.py
+++ b/web/insanityweb/views.py
@@ -69,6 +69,21 @@ def matrix_view(request, testrun_id):
totalnb = testsinst.count()
res = list(testsinst[offset:offset+limit])
+ error_summary = {}
+ for t in res:
+ for r in t.checklist.all().select_related(depth=1):
+ if r.failure:
+ if r.name.id not in error_summary:
+ error_summary[r.name.id] = {
+ 'name': r.name.name,
+ 'description': r.name.description,
+ 'count': 0
+ }
+ error_summary[r.name.id]['count'] += 1
+
+ error_summary = error_summary.values()
+ error_summary.sort(key=lambda x: x['count'], reverse=True)
+
if totalnb != 0 and res != []:
v = Test.objects.values_list("type",flat=True).filter(id__in=(x.id for x in res)).distinct()
@@ -125,7 +140,8 @@ def matrix_view(request, testrun_id):
'crashonly':int(crashonly),
'timedoutonly':int(timedoutonly),
"offset":offset,
- "limit":limit
+ "limit":limit,
+ 'errorsummary': error_summary
})
def handler404(request):
diff --git a/web/templates/insanityweb/matrix_view.html b/web/templates/insanityweb/matrix_view.html
index 2790d84..9ffa067 100644
--- a/web/templates/insanityweb/matrix_view.html
+++ b/web/templates/insanityweb/matrix_view.html
@@ -68,6 +68,26 @@ Matrix View of Testrun #{{testrun.id}}
</td>
</tr>
</table>
+
+ <h3>Error summary</h3>
+ <table>
+ <thead>
+ <tr>
+ <th>Check item</th>
+ <th>Failures</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ {% for item in errorsummary %}
+ <tr>
+ <td>{{ item.name }}</td>
+ <td class="result-FAIL">{{ item.count }}</td>
+ <td>{{ item.description }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
{% endfor %}
{% endblock %}
diff --git a/web/templates/insanityweb/test_checklist_dict.html b/web/templates/insanityweb/test_checklist_dict.html
index 8cce35f..9376e1b 100644
--- a/web/templates/insanityweb/test_checklist_dict.html
+++ b/web/templates/insanityweb/test_checklist_dict.html
@@ -16,7 +16,7 @@
{% if result.success %}OK{% else %}Fail{% if result.expected_failure %} (expected){% endif %}{% endif %}
</td>
{% endif %}
- <td>{{result.name.description}}</td>
+ <td>{{result.name.description}}{% if not result.success and result.explanation %} <em>(likely reason: {{ result.explanation }})</em>{% endif %}</td>
</tr>
{% endfor %}
</table>