summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThibault Saunier <tsaunier@gnome.org>2014-11-28 21:42:47 (GMT)
committerThibault Saunier <tsaunier@gnome.org>2014-12-05 11:17:40 (GMT)
commit71566688bff2b1593ffb93d575ec03f35e9392a8 (patch)
tree8af52511f6a946ea71c904bd38448cd8b56d1485
parent36aed39259c27a07bf61290fd536b395942ab1de (diff)
downloadgst-devtools-71566688bff2b1593ffb93d575ec03f35e9392a8.tar.gz
gst-devtools-71566688bff2b1593ffb93d575ec03f35e9392a8.tar.xz
validate: Remove the default testsuite implementation
The default testsuite implementation should belong to the default asset repo where we have the corresponding knowledge. We should style manage a sensible list of known blacklisted tests, encoding profiles, and generators in GstValidate itself and allow testsuite actual implementations to easily use them though the register_default_* methods. This allow us to be able to remove the ugly execfile() call.
-rw-r--r--validate/launcher/apps/Makefile.am2
-rw-r--r--validate/launcher/apps/gstvalidate.py141
-rw-r--r--validate/launcher/apps/validate/Makefile.am3
-rw-r--r--validate/launcher/apps/validate/validate_testsuite.py219
-rw-r--r--validate/launcher/main.py9
-rw-r--r--validate/launcher/utils.py3
6 files changed, 146 insertions, 231 deletions
diff --git a/validate/launcher/apps/Makefile.am b/validate/launcher/apps/Makefile.am
index 34687c3..4bf6528 100644
--- a/validate/launcher/apps/Makefile.am
+++ b/validate/launcher/apps/Makefile.am
@@ -1,6 +1,6 @@
appsdir = $(libdir)/gst-validate-launcher/python/launcher/apps/
-SUBDIRS = validate
+SUBDIRS =
apps_PYTHON = \
gstvalidate.py
diff --git a/validate/launcher/apps/gstvalidate.py b/validate/launcher/apps/gstvalidate.py
index b6bea15..899ce91 100644
--- a/validate/launcher/apps/gstvalidate.py
+++ b/validate/launcher/apps/gstvalidate.py
@@ -458,8 +458,6 @@ class GstValidateTestManager(GstValidateBaseTestManager):
self._uris = []
self._run_defaults = True
self._is_populated = False
- execfile(os.path.join(os.path.dirname(__file__), "apps",
- "validate", "validate_testsuite.py"), globals())
def init(self):
if which(GST_VALIDATE_COMMAND) and which(GST_VALIDATE_TRANSCODING_COMMAND):
@@ -615,7 +613,140 @@ not been tested and explicitely activated if you set use --wanted-tests ALL""")
super(GstValidateTestManager, self).set_settings(
options, args, reporter)
+ def register_defaults(self):
+ """
+ Registers the defaults:
+ * Scenarios to be used
+ * Encoding formats to be used
+ * Blacklisted tests
+ * Test generators
+ """
+ self.register_default_scenarios()
+ self.register_default_encoding_formats()
+ self.register_default_blacklist()
+ self.register_default_test_generators()
-def gst_validate_checkout_element_present(element_name):
- null = open(os.devnull)
- return subprocess.call("gst-inspect-1.0 videmixer", shell=True, stdout=null, stderr=null)
+ def register_default_scenarios(self):
+ """
+ Registers default test scenarios
+ """
+ if self.options.long_limit != 0:
+ self.add_scenarios([
+ "play_15s",
+ "reverse_playback",
+ "fast_forward",
+ "seek_forward",
+ "seek_backward",
+ "seek_with_stop",
+ "switch_audio_track",
+ "switch_audio_track_while_paused",
+ "switch_subtitle_track",
+ "switch_subtitle_track_while_paused",
+ "disable_subtitle_track_while_paused",
+ "change_state_intensive",
+ "scrub_forward_seeking"])
+ else:
+ self.add_scenarios([
+ "play_15s",
+ "reverse_playback",
+ "fast_forward",
+ "seek_forward",
+ "seek_backward",
+ "seek_with_stop",
+ "switch_audio_track",
+ "switch_audio_track_while_paused",
+ "switch_subtitle_track",
+ "switch_subtitle_track_while_paused",
+ "disable_subtitle_track_while_paused",
+ "change_state_intensive",
+ "scrub_forward_seeking"])
+
+ def register_default_encoding_formats(self):
+ """
+ Registers default encoding formats
+ """
+ self.add_encoding_formats([
+ MediaFormatCombination("ogg", "vorbis", "theora"),
+ MediaFormatCombination("webm", "vorbis", "vp8"),
+ MediaFormatCombination("mp4", "mp3", "h264"),
+ MediaFormatCombination("mkv", "vorbis", "h264"),
+ ])
+
+ def register_default_blacklist(self):
+ self.set_default_blacklist([
+ # hls known issues
+ ("validate.hls.playback.fast_forward.*",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=698155"),
+ ("validate.hls.playback.seek_with_stop.*",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=723268"),
+ ("validate.hls.playback.reverse_playback.*",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
+ ("validate.hls.*scrub_forward_seeking.*",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
+ ("validate.hls.*seek_backward.*",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
+ ("validate.hls.*seek_forward.*",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
+ ("validate.hls.*",
+ "FIXME! The HLS tests are not stable enough "
+ "(at least on the server), try again later."),
+
+ # Matroska/WEBM known issues:
+ ("validate.*.reverse_playback.*webm$",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
+ ("validate.*.reverse_playback.*mkv$",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
+ ("validate.*reverse.*Sintel_2010_720p_mkv",
+ "TODO in matroskademux: FIXME: We should build an index during playback or "
+ "when scanning that can be used here. The reverse playback code requires "
+ " seek_index and seek_entry to be set!"),
+ ("validate.http.playback.seek_with_stop.*webm",
+ "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
+ ("validate.http.playback.seek_with_stop.*mkv",
+ "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
+
+ # MPEG TS known issues:
+ ('(?i)validate.*.playback.reverse_playback.*(?:_|.)(?:|m)ts$',
+ "https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
+ ('validate.file.transcode.to_vorbis_and_vp8_in_webm.GH1_00094_1920x1280_MTS',
+ 'Got error: Internal data stream error. -- Debug message: mpegtsbase.c(1371):'
+ 'mpegts_base_loop (): ...: stream stopped, reason not-negotiated'),
+
+ # HTTP known issues:
+ ("validate.http.*scrub_forward_seeking.*",
+ "This is not stable enough for now."),
+ ("validate.http.playback.change_state_intensive.raw_video_mov",
+ "This is not stable enough for now. (flow return from pad push doesn't match expected value)"),
+
+ # MXF known issues"
+ (".*reverse_playback.*mxf",
+ "Reverse playback is not handled in MXF"),
+ ("validate\.file\.transcode.*mxf",
+ "FIXME: Transcoding and mixing tests need to be tested"),
+
+ # Subtitles known issues
+ ("validate.file.playback.switch_subtitle_track.Sintel_2010_720p_mkv",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=734051"),
+
+ # Videomixing known issues
+ ("validate.file.*.simple.scrub_forward_seeking.synchronized",
+ "https://bugzilla.gnome.org/show_bug.cgi?id=734060"),
+
+ # FLAC known issues"
+ (".*reverse_playback.*flac",
+ "Reverse playback is not handled in flac"),
+
+ # WMV known issues"
+ (".*reverse_playback.*wmv",
+ "Reverse playback is not handled in wmv"),
+ (".*reverse_playback.*asf",
+ "Reverse playback is not handled in asf"),
+ ])
+
+ def register_default_test_generators(self):
+ """
+ Registers default test generators
+ """
+ self.add_generators([GstValidatePlaybinTestsGenerator(self),
+ GstValidateMediaCheckTestsGenerator(self),
+ GstValidateTranscodingTestsGenerator(self)])
diff --git a/validate/launcher/apps/validate/Makefile.am b/validate/launcher/apps/validate/Makefile.am
deleted file mode 100644
index 7c0a4e9..0000000
--- a/validate/launcher/apps/validate/Makefile.am
+++ /dev/null
@@ -1,3 +0,0 @@
-appsdir = $(libdir)/gst-validate-launcher/python/launcher/apps/validate
-
-apps_PYTHON = validate_testsuite.py
diff --git a/validate/launcher/apps/validate/validate_testsuite.py b/validate/launcher/apps/validate/validate_testsuite.py
deleted file mode 100644
index f80ad3a..0000000
--- a/validate/launcher/apps/validate/validate_testsuite.py
+++ /dev/null
@@ -1,219 +0,0 @@
-#!/usr/bin/env python2
-#
-# validate_default_testsuite.py
-#
-# Copyright (c) 2014, Thibault Saunier tsaunier@gnome.org
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this program; if not, write to the
-# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
-# Boston, MA 02110-1301, USA.
-
-
-valid_mixing_scenarios = ["play_15s",
- "fast_forward",
- "seek_forward",
- "seek_backward",
- "seek_with_stop",
- "scrub_forward_seeking"]
-
-
-def register_compositing_tests(self):
- """
- Those tests are not activated in the default testsuite,
- they should be activated in a configuration file.
- """
- for compositor in ["compositor", "glmixer"]:
- if gst_validate_checkout_element_present(compositor):
- self.add_generators(
- GstValidateMixerTestsGenerator(compositor, self,
- compositor,
- "video",
- converter="deinterlace ! videoconvert ! videorate ! videoscale ! video/x-raw,framerate=25/1,pixel-aspect-ratio=1/1",
- valid_scenarios=valid_mixing_scenarios))
-
-
-def register_default_test_generators(self):
- """
- Registers default test generators
- """
- self.add_generators([GstValidatePlaybinTestsGenerator(self),
- GstValidateMediaCheckTestsGenerator(self),
- GstValidateTranscodingTestsGenerator(self)])
-
- for compositor in ["compositor", "glvideomixer"]:
- self.add_generators(
- GstValidateMixerTestsGenerator(compositor + ".simple", self,
- compositor,
- "video",
- converter="deinterlace ! videoconvert",
- mixed_srcs={
- "synchronized": {"mixer_props": "sink_1::alpha=0.5 sink_1::xpos=50 sink_1::ypos=50",
- "sources":
- ("videotestsrc pattern=snow timestamp-offset=3000000000 ! 'video/x-raw,format=AYUV,width=640,height=480,framerate=(fraction)30/1' ! timeoverlay",
- "videotestsrc pattern=smpte ! 'video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1' ! timeoverlay")},
- "bgra": ("videotestsrc ! video/x-raw, framerate=\(fraction\)10/1, width=100, height=100",
- "videotestsrc ! video/x-raw, framerate=\(fraction\)5/1, width=320, height=240")
- },
- valid_scenarios=valid_mixing_scenarios))
-
- self.add_generators(
- GstValidateMixerTestsGenerator("audiomixer.simple", self,
- "audiomixer",
- "audio",
- converter="audioconvert ! audioresample",
- mixed_srcs={
- "basic": {"mixer_props": "",
- "sources":
- ("audiotestsrc wave=triangle",
- "audiotestsrc wave=ticks")},
- },
- valid_scenarios=valid_mixing_scenarios))
-
-
-def register_default_scenarios(self):
- """
- Registers default test scenarios
- """
- if self.options.long_limit != 0:
- self.add_scenarios([
- "play_15s",
- "reverse_playback",
- "fast_forward",
- "seek_forward",
- "seek_backward",
- "seek_with_stop",
- "switch_audio_track",
- "switch_audio_track_while_paused",
- "switch_subtitle_track",
- "switch_subtitle_track_while_paused",
- "disable_subtitle_track_while_paused",
- "change_state_intensive",
- "scrub_forward_seeking"])
- else:
- self.add_scenarios([
- "play_15s",
- "reverse_playback",
- "fast_forward",
- "seek_forward",
- "seek_backward",
- "seek_with_stop",
- "switch_audio_track",
- "switch_audio_track_while_paused",
- "switch_subtitle_track",
- "switch_subtitle_track_while_paused",
- "disable_subtitle_track_while_paused",
- "change_state_intensive",
- "scrub_forward_seeking"])
-
-
-def register_default_encoding_formats(self):
- """
- Registers default encoding formats
- """
- self.add_encoding_formats([
- MediaFormatCombination("ogg", "vorbis", "theora"),
- MediaFormatCombination("webm", "vorbis", "vp8"),
- MediaFormatCombination("mp4", "mp3", "h264"),
- MediaFormatCombination("mkv", "vorbis", "h264"),
- ])
-
-
-def register_default_blacklist(self):
- self.set_default_blacklist([
- # hls known issues
- ("validate.hls.playback.fast_forward.*",
- "https://bugzilla.gnome.org/show_bug.cgi?id=698155"),
- ("validate.hls.playback.seek_with_stop.*",
- "https://bugzilla.gnome.org/show_bug.cgi?id=723268"),
- ("validate.hls.playback.reverse_playback.*",
- "https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
- ("validate.hls.*scrub_forward_seeking.*",
- "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
- ("validate.hls.*seek_backward.*",
- "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
- ("validate.hls.*seek_forward.*",
- "https://bugzilla.gnome.org/show_bug.cgi?id=606382"),
-
- # Matroska/WEBM known issues:
- ("validate.*.reverse_playback.*webm$",
- "https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
- ("validate.*.reverse_playback.*mkv$",
- "https://bugzilla.gnome.org/show_bug.cgi?id=679250"),
- ("validate.*reverse.*Sintel_2010_720p_mkv",
- "TODO in matroskademux: FIXME: We should build an index during playback or "
- "when scanning that can be used here. The reverse playback code requires "
- " seek_index and seek_entry to be set!"),
- ("validate.http.playback.seek_with_stop.*webm",
- "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
- ("validate.http.playback.seek_with_stop.*mkv",
- "matroskademux.gst_matroska_demux_handle_seek_push: Seek end-time not supported in streaming mode"),
-
- # MPEG TS known issues:
- ('(?i)validate.*.playback.reverse_playback.*(?:_|.)(?:|m)ts$',
- "https://bugzilla.gnome.org/show_bug.cgi?id=702595"),
- ('validate.file.transcode.to_vorbis_and_vp8_in_webm.GH1_00094_1920x1280_MTS',
- 'Got error: Internal data stream error. -- Debug message: mpegtsbase.c(1371):'
- 'mpegts_base_loop (): ...: stream stopped, reason not-negotiated'),
-
- # HTTP known issues:
- ("validate.http.*scrub_forward_seeking.*",
- "This is not stable enough for now."),
- ("validate.http.playback.change_state_intensive.raw_video_mov",
- "This is not stable enough for now. (flow return from pad push doesn't match expected value)"),
-
- # MXF known issues"
- (".*reverse_playback.*mxf", "Reverse playback is not handled in MXF"),
- ("validate\.file\.transcode.*mxf",
- "FIXME: Transcoding and mixing tests need to be tested"),
-
- # Subtitles known issues
- ("validate.file.playback.switch_subtitle_track.Sintel_2010_720p_mkv",
- "https://bugzilla.gnome.org/show_bug.cgi?id=734051"),
-
- # Videomixing known issues
- ("validate.file.*.simple.scrub_forward_seeking.synchronized",
- "https://bugzilla.gnome.org/show_bug.cgi?id=734060"),
-
- # FLAC known issues"
- (".*reverse_playback.*flac",
- "Reverse playback is not handled in flac"),
-
- # WMV known issues"
- (".*reverse_playback.*wmv", "Reverse playback is not handled in wmv"),
- (".*reverse_playback.*asf", "Reverse playback is not handled in asf"),
- ])
-
-
-def register_defaults(self):
- self.register_default_scenarios()
- self.register_default_encoding_formats()
- self.register_default_blacklist()
- self.register_default_test_generators()
-
-
-def register_all(self):
- self.register_defaults()
- self.register_compositing_tests()
-
-
-try:
- GstValidateTestManager.register_defaults = register_defaults
- GstValidateTestManager.register_all = register_all
- GstValidateTestManager.register_default_blacklist = register_default_blacklist
- GstValidateTestManager.register_default_test_generators = register_default_test_generators
- GstValidateTestManager.register_default_scenarios = register_default_scenarios
- GstValidateTestManager.register_compositing_tests = register_compositing_tests
- GstValidateTestManager.register_default_encoding_formats = register_default_encoding_formats
-except NameError:
- pass
diff --git a/validate/launcher/main.py b/validate/launcher/main.py
index d47b411..af6b50f 100644
--- a/validate/launcher/main.py
+++ b/validate/launcher/main.py
@@ -131,6 +131,10 @@ http://wiki.pitivi.org/wiki/Bug_reporting#Debug_logs).
QA_ASSETS = "gst-qa-assets"
MEDIAS_FOLDER = "medias"
DEFAULT_GST_QA_ASSETS_REPO = "git://people.freedesktop.org/~tsaunier/gst-qa-assets/"
+DEFAULT_VALIDATE_TESTSUITE = os.path.join(DEFAULT_MAIN_DIR,
+ QA_ASSETS,
+ "testsuites",
+ "gst-validate-default.testsuite")
def update_assets(options):
@@ -233,10 +237,11 @@ def main(libsdir):
"It implies --generate-media-info but enabling frame detection")
parser.add_argument("-lt", "--long-test-limit", dest="long_limit",
default=utils.LONG_TEST, action='store',
- help="Defines the limit from which a test is considered as long (in seconds)",
+ help="Defines the limit from which a test is considered as long (in seconds)"
+ " note that 0 will enable all tests",
type=int),
parser.add_argument("-c", "--config", dest="config",
- default=None,
+ default=DEFAULT_VALIDATE_TESTSUITE,
help="""Lets you specify a file where the testsuite to execute is defined.
In this file you will have acces to the TestManager objects that you can configure with
its various methods. For example you can find the 'validate' variable, in case the GstValidateManager
diff --git a/validate/launcher/utils.py b/validate/launcher/utils.py
index 286dd8c..d5c5c6c 100644
--- a/validate/launcher/utils.py
+++ b/validate/launcher/utils.py
@@ -30,7 +30,7 @@ from operator import itemgetter
GST_SECOND = long(1000000000)
DEFAULT_TIMEOUT = 30
-DEFAULT_MAIN_DIR = os.path.expanduser("~/gst-validate/")
+DEFAULT_MAIN_DIR = os.path.join(os.path.expanduser("~"), "gst-validate")
DEFAULT_GST_QA_ASSETS = os.path.join(DEFAULT_MAIN_DIR, "gst-qa-assets")
DISCOVERER_COMMAND = "gst-discoverer-1.0"
# Use to set the duration from which a test is concidered as being 'long'
@@ -44,6 +44,7 @@ class Result(object):
PASSED = "Passed"
KNOWN_ERROR = "Known error"
+
class Protocols(object):
HTTP = "http"
FILE = "file"