VirtualBox

Changeset 97267 in vbox for trunk/src


Ignore:
Timestamp:
Oct 24, 2022 12:09:44 AM (3 years ago)
Author:
vboxsync
svn:sync-xref-src-repo-rev:
154255
Message:

ValKit/analysis: Adding better filtering and some documentation (--help) for the analyze tool. Some improvements for --option[:=]value parsing too.

Location:
trunk/src/VBox/ValidationKit/analysis
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/src/VBox/ValidationKit/analysis/analyze.py

    r97266 r97267  
    4040__version__ = "$Revision$"
    4141
    42 
    43 import os.path
    44 import sys
     42# Standard python imports.
     43import re;
     44import os;
     45import textwrap;
     46import sys;
    4547
    4648# Only the main script needs to modify the path.
    … …  
    5153
    5254# Validation Kit imports.
    53 from analysis import reader    ## @todo fix testanalysis/__init__.py.
     55from analysis import reader
    5456from analysis import reporting
    55 #from analysis import diff
    5657
    5758
    5859def usage():
    59     """ Display usage """
    60     print('usage: %s [options] <test1.xml/txt> <test2.xml/txt> [..] [-- <baseline1.file> [baseline2.file] [..]]'
    61           % (sys.argv[0]));
    62     print('')
    63     print('options:')
    64     print('  --filter <test-sub-string>')
     60    """
     61    Display usage.
     62    """
     63    # Set up the output wrapper.
     64    try:    cCols = os.get_terminal_size()[0] # since 3.3
     65    except: cCols = 79;
     66    oWrapper = textwrap.TextWrapper(width = cCols);
     67
     68    # Do the outputting.
     69    print('Tool for comparing test results.');
     70    print('');
     71    oWrapper.subsequent_indent = ' ' * (len('usage: ') + 4);
     72    print(oWrapper.fill('usage: analyze.py [options] [collection-1] -- [collection-2] [-- [collection3] [..]])'))
     73    oWrapper.subsequent_indent = '';
     74    print('');
     75    print(oWrapper.fill('This tool compares two or more result collections, using one as a baseline (first by default) '
     76                        'and showing how the results in others differs from it.'));
     77    print('');
     78    print(oWrapper.fill('The results (XML file) from one or more test runs makes up a collection.  A collection can be '
     79                        'named using the --name <name> option, or will get a sequential name automatically.  The baseline '
     80                        'collection will have "(baseline)" appended to its name.'));
     81    print('');
     82    print(oWrapper.fill('A test run produces one XML file, either via the testdriver/reporter.py machinery or via the IPRT '
     83                        'test.cpp code. In the latter case it can be enabled and controlled via IPRT_TEST_FILE.  A collection '
     84                        'consists of one or more of test runs (i.e. XML result files).  These are combined (aka distilled) '
     85                        'into a single set of results before comparing them with the others.  The --best and --avg options '
     86                        'controls how this combining is done.  The need for this is mainly to try counteract some of the '
     87                        'instability typically found in the restuls.  Just because one test run produces a better result '
     88                        'after a change does not necessarily mean this will always be the case and that the change was to '
     89                        'the better, it might just have been regular fluctuations in the test results.'));
     90
     91    oWrapper.initial_indent    = '      ';
     92    oWrapper.subsequent_indent = '      ';
     93    print('');
     94    print('Options governing combining (distillation):');
     95    print('  --avg, --average');
     96    print(oWrapper.fill('Picks the best result by calculating the average values across all the runs.'));
     97    print('');
     98    print('  --best');
     99    print(oWrapper.fill('Picks the best result from all the runs.  For values, this means making guessing what result is '
     100                        'better based on the unit.  This may not always lead to the right choices.'));
     101    print(oWrapper.initial_indent + 'Default: --best');
     102
     103    print('');
     104    print('Options relating to collections:');
     105    print('  --name <name>');
     106    print(oWrapper.fill('Sets the name of the current collection.  By default a collection gets a sequential number.'));
     107    print('');
     108    print('  --baseline <num>');
     109    print(oWrapper.fill('Sets collection given by <num> (0-based) as the baseline collection.'));
     110    print(oWrapper.initial_indent + 'Default: --baseline 0')
     111
     112    print('');
     113    print('Filtering options:');
     114    print('  --filter-test <substring>');
     115    print(oWrapper.fill('Exclude tests not containing any of the substrings given via the --filter-test option.  The '
     116                        'matching is done with full test name, i.e. all parent names are prepended with ", " as separator '
     117                        '(for example "tstIOInstr, CPUID EAX=1").'));
     118    print('');
     119    print('  --filter-test-out <substring>');
     120    print(oWrapper.fill('Exclude tests containing the given substring.  As with --filter-test, the matching is done against '
     121                        'the full test name.'));
     122    print('');
     123    print('  --filter-value <substring>');
     124    print(oWrapper.fill('Exclude values not containing any of the substrings given via the --filter-value option.  The '
     125                        'matching is done against the value name prefixed by the full test name and ": " '
     126                        '(for example "tstIOInstr, CPUID EAX=1: real mode, CPUID").'));
     127    print('');
     128    print('  --filter-value-out <substring>');
     129    print(oWrapper.fill('Exclude value containing the given substring.  As with --filter-value, the matching is done against '
     130                        'the value name prefixed by the full test name.'));
     131
     132    print('');
     133    print('  --regex-test <expr>');
     134    print(oWrapper.fill('Same as --filter-test except the substring matching is done via a regular expression.'));
     135    print('');
     136    print('  --regex-test-out <expr>');
     137    print(oWrapper.fill('Same as --filter-test-out except the substring matching is done via a regular expression.'));
     138    print('');
     139    print('  --regex-value <expr>');
     140    print(oWrapper.fill('Same as --filter-value except the substring matching is done via a regular expression.'));
     141    print('');
     142    print('  --regex-value-out <expr>');
     143    print(oWrapper.fill('Same as --filter-value-out except the substring matching is done via a regular expression.'));
     144    print('');
     145    print('  --filter-out-empty-leaf-tests');
     146    print(oWrapper.fill('Removes any leaf tests that are without any values or sub-tests.  This is useful when '
     147                        'only considering values, especially when doing additional value filtering.'));
     148
     149    print('');
     150    print('Output options:');
     151    print('  --brief, --verbose');
     152    print(oWrapper.fill('Whether to omit (--brief) the value for non-baseline runs and just get along with the difference.'));
     153    print(oWrapper.initial_indent + 'Default: --brief');
     154    print('');
     155    print('  --pct <num>, --pct-precision <num>');
     156    print(oWrapper.fill('Specifies the number of decimal place to use when formatting the difference as percent.'));
     157    print(oWrapper.initial_indent + 'Default: --pct 2');
    65158    return 1;
     159
    66160
    67161class ResultCollection(object):
    … …  
    94188    def filterTests(self, asFilters):
    95189        """
    96         Filters all the test trees using asFilters.
     190        Keeps all the tests in the test trees sub-string matching asFilters (str or re).
    97191        """
    98192        for oTestTree in self.aoTestTrees:
    99193            oTestTree.filterTests(asFilters);
     194        return self;
     195
     196    def filterOutTests(self, asFilters):
     197        """
     198        Removes all the tests in the test trees sub-string matching asFilters (str or re).
     199        """
     200        for oTestTree in self.aoTestTrees:
     201            oTestTree.filterOutTests(asFilters);
     202        return self;
     203
     204    def filterValues(self, asFilters):
     205        """
     206        Keeps all the tests in the test trees sub-string matching asFilters (str or re).
     207        """
     208        for oTestTree in self.aoTestTrees:
     209            oTestTree.filterValues(asFilters);
     210        return self;
     211
     212    def filterOutValues(self, asFilters):
     213        """
     214        Removes all the tests in the test trees sub-string matching asFilters (str or re).
     215        """
     216        for oTestTree in self.aoTestTrees:
     217            oTestTree.filterOutValues(asFilters);
     218        return self;
     219
     220    def filterOutEmptyLeafTests(self):
     221        """
     222        Removes all the tests in the test trees that have neither child tests nor values.
     223        """
     224        for oTestTree in self.aoTestTrees:
     225            oTestTree.filterOutEmptyLeafTests();
    100226        return self;
    101227
    … …  
    145271
    146272
     273# matchWithValue hacks.
     274g_asOptions = [];
     275g_iOptInd   = 1;
     276g_sOptArg   = '';
     277
     278def matchWithValue(sOption):
     279    """ Matches an option with a value, placing the value in g_sOptArg if it matches. """
     280    global g_asOptions, g_iOptInd, g_sOptArg;
     281    sArg = g_asOptions[g_iOptInd];
     282    if sArg.startswith(sOption):
     283        if len(sArg) == len(sOption):
     284            if g_iOptInd + 1 < len(g_asOptions):
     285                g_iOptInd += 1;
     286                g_sOptArg  = g_asOptions[g_iOptInd];
     287                return True;
     288
     289            print('syntax error: Option %s takes a value!' % (sOption,));
     290            raise Exception('syntax error: Option %s takes a value!' % (sOption,));
     291
     292        if sArg[len(sOption)] in ('=', ':'):
     293            g_sOptArg = sArg[len(sOption) + 1:];
     294            return True;
     295    return False;
    147296
    148297
    … …  
    152301    # Parse arguments
    153302    #
    154     oCurCollection      = ResultCollection('#0');
    155     aoCollections       = [ oCurCollection, ];
    156     iBaseline           = 0;
    157     sDistillationMethod = 'best';
    158     fBrief              = True;
    159     cPctPrecision       = 2;
    160     asFilters           = [];
    161 
    162     iArg = 1;
    163     while iArg < len(asArgs):
    164         #print("dbg: iArg=%s '%s'" % (iArg, asArgs[iArg],));
    165         if asArgs[iArg].startswith('--help'):
     303    oCurCollection          = ResultCollection('#0');
     304    aoCollections           = [ oCurCollection, ];
     305    iBaseline               = 0;
     306    sDistillationMethod     = 'best';
     307    fBrief                  = True;
     308    cPctPrecision           = 2;
     309    asTestFilters           = [];
     310    asTestOutFilters        = [];
     311    asValueFilters          = [];
     312    asValueOutFilters       = [];
     313    fFilterOutEmptyLeafTest = True;
     314
     315    global g_asOptions, g_iOptInd, g_sOptArg;
     316    g_asOptions = asArgs;
     317    g_iOptInd   = 1;
     318    while g_iOptInd < len(asArgs):
     319        sArg      = asArgs[g_iOptInd];
     320        g_sOptArg = '';
     321        #print("dbg: g_iOptInd=%s '%s'" % (g_iOptInd, sArg,));
     322
     323        if sArg.startswith('--help'):
    166324            return usage();
    167         if asArgs[iArg] == '--filter':
    168             iArg += 1;
    169             asFilters.append(asArgs[iArg]);
    170         elif asArgs[iArg] == '--best':
     325
     326        if matchWithValue('--filter-test'):
     327            asTestFilters.append(g_sOptArg);
     328        elif matchWithValue('--filter-test-out'):
     329            asTestOutFilters.append(g_sOptArg);
     330        elif matchWithValue('--filter-value'):
     331            asValueFilters.append(g_sOptArg);
     332        elif matchWithValue('--filter-value-out'):
     333            asValueOutFilters.append(g_sOptArg);
     334
     335        elif matchWithValue('--regex-test'):
     336            asTestFilters.append(re.compile(g_sOptArg));
     337        elif matchWithValue('--regex-test-out'):
     338            asTestOutFilters.append(re.compile(g_sOptArg));
     339        elif matchWithValue('--regex-value'):
     340            asValueFilters.append(re.compile(g_sOptArg));
     341        elif matchWithValue('--regex-value-out'):
     342            asValueOutFilters.append(re.compile(g_sOptArg));
     343
     344        elif sArg == '--filter-out-empty-leaf-tests':
     345            fFilterOutEmptyLeafTest = True;
     346        elif sArg == '--no-filter-out-empty-leaf-tests':
     347            fFilterOutEmptyLeafTest = False;
     348
     349        elif sArg == '--best':
    171350            sDistillationMethod = 'best';
    172         elif asArgs[iArg] in ('--avg', '--average'):
     351        elif sArg in ('--avg', '--average'):
    173352            sDistillationMethod = 'avg';
    174         elif asArgs[iArg] == '--brief':
     353
     354        elif sArg == '--brief':
    175355            fBrief = True;
    176         elif asArgs[iArg] == '--verbose':
     356        elif sArg == '--verbose':
    177357            fBrief = False;
    178         elif asArgs[iArg] in ('--pct', '--pct-precision'):
    179             iArg += 1;
    180             cPctPrecision = int(asArgs[iArg]);
    181         elif asArgs[iArg] in ('--base', '--baseline'):
    182             iArg += 1;
    183             iBaseline = int(asArgs[iArg]);
     358
     359        elif matchWithValue('--pct') or matchWithValue('--pct-precision'):
     360            cPctPrecision = int(g_sOptArg);
     361        elif matchWithValue('--base') or matchWithValue('--baseline'):
     362            iBaseline = int(g_sOptArg);
     363
    184364        # '--' starts a new collection.  If current one is empty, drop it.
    185         elif asArgs[iArg] == '--':
     365        elif sArg == '--':
    186366            print("dbg: new collection");
    187367            #if oCurCollection.isEmpty():
    … …  
    189369            oCurCollection = ResultCollection("#%s" % (len(aoCollections),));
    190370            aoCollections.append(oCurCollection);
     371
    191372        # Name the current result collection.
    192         elif asArgs[iArg] == '--name':
    193             iArg += 1;
    194             oCurCollection.sName = asArgs[iArg];
     373        elif matchWithValue('--name'):
     374            oCurCollection.sName = g_sOptArg;
     375
    195376        # Read in a file and add it to the current data set.
    196377        else:
    197             if not oCurCollection.append(asArgs[iArg]):
     378            if not oCurCollection.append(sArg):
    198379                return 1;
    199         iArg += 1;
     380        g_iOptInd += 1;
    200381
    201382    #
    … …  
    218399
    219400    #
    220     # Apply filtering before distilling each collection into a single result
    221     # tree and comparing them to the first one.
    222     #
    223     if asFilters:
     401    # Apply filtering before distilling each collection into a single result tree.
     402    #
     403    if asTestFilters:
    224404        for oCollection in aoCollections:
    225             oCollection.filterTests(asFilters);
    226 
     405            oCollection.filterTests(asTestFilters);
     406    if asTestOutFilters:
     407        for oCollection in aoCollections:
     408            oCollection.filterOutTests(asTestOutFilters);
     409
     410    if asValueFilters:
     411        for oCollection in aoCollections:
     412            oCollection.filterValues(asValueFilters);
     413    if asValueOutFilters:
     414        for oCollection in aoCollections:
     415            oCollection.filterOutValues(asValueOutFilters);
     416
     417    if fFilterOutEmptyLeafTest:
     418        for oCollection in aoCollections:
     419            oCollection.filterOutEmptyLeafTests();
     420
     421    # Distillation.
    227422    for oCollection in aoCollections:
    228423        oCollection.distill(sDistillationMethod);

  • trunk/src/VBox/ValidationKit/analysis/reader.py

    r97266 r97267  
    4646import datetime;
    4747import os;
     48import re;
    4849import sys;
    4950import traceback;
    … …  
    135136        return Value(oParentTest, self.sName, self.sUnit, self.sTimestamp, self.lValue);
    136137
     138    def matchFilters(self, sPrefix, aoFilters):
     139        """
     140        Checks for any substring match between aoFilters (str or re.Pattern)
     141        and the value name prefixed by sPrefix.
     142
     143        Returns True if any of the filters matches.
     144        Returns False if none of the filters matches.
     145        """
     146        sFullName = sPrefix + self.sName;
     147        for oFilter in aoFilters:
     148            if oFilter.search(sFullName) is not None if isinstance(oFilter, re.Pattern) else sFullName.find(oFilter) >= 0:
     149                return True;
     150        return False;
     151
    137152
    138153    @staticmethod
    … …  
    306321        return self.getFullNameWorker(cSkipUpper)[0];
    307322
    308     def matchFilters(self, asFilters):
    309         """
    310         Checks if the all of the specified filter strings are substrings
    311         of the full test name.  Returns True / False.
    312         """
    313         sName = self.getFullName();
    314         for sFilter in asFilters:
    315             if sName.find(sFilter) < 0:
    316                 return False;
    317         return True;
     323    def matchFilters(self, aoFilters):
     324        """
     325        Checks for any substring match between aoFilters (str or re.Pattern)
     326        and the full test name.
     327
     328        Returns True if any of the filters matches.
     329        Returns False if none of the filters matches.
     330        """
     331        sFullName = self.getFullName();
     332        for oFilter in aoFilters:
     333            if oFilter.search(sFullName) is not None if isinstance(oFilter, re.Pattern) else sFullName.find(oFilter) >= 0:
     334                return True;
     335        return False;
    318336
    319337    # manipulation
    320338
    321     def filterTestsWorker(self, asFilters):
     339    def filterTestsWorker(self, asFilters, fReturnOnMatch):
    322340        # depth first
    323341        i = 0;
    324342        while i < len(self.aoChildren):
    325             if self.aoChildren[i].filterTestsWorker(asFilters):
     343            if self.aoChildren[i].filterTestsWorker(asFilters, fReturnOnMatch):
    326344                i += 1;
    327345            else:
    … …  
    332350        if self.aoChildren:
    333351            return True;
    334         return self.matchFilters(asFilters);
     352        if self.matchFilters(asFilters):
     353            return fReturnOnMatch;
     354        return not fReturnOnMatch;
    335355
    336356    def filterTests(self, asFilters):
     357        """ Keep tests matching asFilters. """
    337358        if asFilters:
    338             self.filterTestsWorker(asFilters)
     359            self.filterTestsWorker(asFilters, True);
    339360        return self;
     361
     362    def filterOutTests(self, asFilters):
     363        """ Removes tests matching asFilters. """
     364        if asFilters:
     365            self.filterTestsWorker(asFilters, False);
     366        return self;
     367
     368    def filterValuesWorker(self, asFilters, fKeepWhen):
     369        # Process children recursively.
     370        for oChild in self.aoChildren:
     371            oChild.filterValuesWorker(asFilters, fKeepWhen);
     372
     373        # Filter our values.
     374        iValue = len(self.aoValues);
     375        if iValue > 0:
     376            sFullname = self.getFullName() + ': ';
     377            while iValue > 0:
     378                iValue -= 1;
     379                if self.aoValues[iValue].matchFilters(sFullname, asFilters) != fKeepWhen:
     380                    del self.aoValues[iValue];
     381        return None;
     382
     383    def filterValues(self, asFilters):
     384        """ Keep values matching asFilters. """
     385        if asFilters:
     386            self.filterValuesWorker(asFilters, True);
     387        return self;
     388
     389    def filterOutValues(self, asFilters):
     390        """ Removes values matching asFilters. """
     391        if asFilters:
     392            self.filterValuesWorker(asFilters, False);
     393        return self;
     394
     395    def filterOutEmptyLeafTests(self):
     396        """
     397        Removes any child tests that has neither values nor sub-tests.
     398        Returns True if leaf, False if not.
     399        """
     400        iChild = len(self.aoChildren);
     401        while iChild > 0:
     402            iChild -= 1;
     403            if self.aoChildren[iChild].filterOutEmptyLeafTests():
     404                del self.aoChildren[iChild];
     405        return not self.aoChildren and not self.aoValues;
    340406
    341407    @staticmethod
Note: See TracChangeset for help on using the changeset viewer.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette