1 """
2 A TestRunner for use with the Python unit testing framework. It
3 generates a HTML report to show the result at a glance.
4
5 ------------------------------------------------------------------------
6 Copyright (c) 2004-2020, Wai Yip Tung
7 All rights reserved.
8 Redistribution and use in source and binary forms, with or without
9 modification, are permitted provided that the following conditions are
10 met:
11 * Redistributions of source code must retain the above copyright notice,
12 this list of conditions and the following disclaimer.
13 * Redistributions in binary form must reproduce the above copyright
14 notice, this list of conditions and the following disclaimer in the
15 documentation and/or other materials provided with the distribution.
16 * Neither the name Wai Yip Tung nor the names of its contributors may be
17 used to endorse or promote products derived from this software without
18 specific prior written permission.
19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
20 IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
22 PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
23 OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
26 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
27 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
28 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 """
31
32 # URL: http://tungwaiyip.info/software/HTMLTestRunner.html
33
34 __author__ = "Wai Yip Tung , bugmaster"
35 __version__ = "0.9.0"
36
37 """
38 Change History
39
40 Version 0.9.0
41 * Increased repeat execution
42 * Added failure screenshots
43
44 Version 0.8.2
45 * Show output inline instead of popup window (Viorel Lupu).
46
47 Version in 0.8.1
48 * Validated XHTML (Wolfgang Borgert).
49 * Added description of test classes and test cases.
50
51 Version in 0.8.0
52 * Define Template_mixin class for customization.
53 * Workaround a IE 6 bug that it does not treat
165
166
167
168
169
170 %(stylesheet)s
171
172 173 355 %(heading)s 356 %(report)s 357 %(ending)s 358 %(chart_script)s 359
360
361 """
362 # variables: (title, generator, stylesheet, heading, report, ending)
363
364 # ------------------------------------------------------------------------
365 # Stylesheet
366 #
367 # alternatively use a for external style sheet, e.g.
368 #
369
370 STYLESHEET_TMPL = """
371
523 """
524
525 # ------------------------------------------------------------------------
526 # Heading
527 #
528
529 HEADING_TMPL = """
530

541
571 """ # variables: (title, parameters, description)
572
573 # ------------------------------------------------------------------------
574 # Pie chart
575 #
576
577 ECHARTS_SCRIPT = """
578
617 """
618
619 HEADING_ATTRIBUTE_TMPL = """%(name)s:%(value)s
620 """ # variables: (name, value)
621
622 # ------------------------------------------------------------------------
623 # Report
624 #
625
626 REPORT_TMPL = """
627

628 Summary 629 Pass 630 Failed 631 Error 632 Skip 633 All 634

635 636 637 638 639 640 641 642 643 644 645 646 647 %(test_list)s 648 649 650 651 652 653 654 655 656 657

Test Group/Test case	Count	Pass	Fail	Error	View	Screenshots
Total	%(count)s	%(Pass)s	%(fail)s	%(error)s

658 """ # variables: (test_list, count, Pass, fail, error)
659
660 REPORT_CLASS_TMPL = r"""
661 662 %(desc)s 663 %(count)s 664 %(Pass)s 665 %(fail)s 666 %(error)s 667 Detail 668   669
670 """ # variables: (style, desc, count, Pass, fail, error, cid)
671
672 REPORT_TEST_WITH_OUTPUT_TMPL = r"""
673 674 675 676 677 678 %(status)s 679 688 689 690 %(img)s 691
692 """ # variables: (tid, Class, style, desc, status)
693
694 REPORT_TEST_NO_OUTPUT_TMPL = r"""
695 696 697 %(status)s 698 %(img)s 699
700 """ # variables: (tid, Class, style, desc, status)
701
702 REPORT_TEST_OUTPUT_TMPL = r"""
703 %(id)s: %(output)s
704 """ # variables: (id, output)
705
706 IMG_TMPL = r"""
707 show
708
713 """
714 # ------------------------------------------------------------------------
715 # ENDING
716 #
717
718 ENDING_TMPL = """"""
719
720
721 # -------------------- The end of the Template class -------------------
722
723
724 TestResult = unittest.TestResult
725
726
727 class _TestResult(TestResult):
728 # note: _TestResult is a pure representation of results.
729 # It lacks the output and reporting ability compares to unittest._TextTestResult.
730
731 def __init__(self, verbosity=1, rerun=0, save_last_run=False):
732 TestResult.__init__(self)
733 self.stdout0 = None
734 self.stderr0 = None
735 self.success_count = 0
736 self.failure_count = 0
737 self.error_count = 0
738 self.skip_count = 0
739 self.verbosity = verbosity
740 self.rerun = rerun
741 self.save_last_run = save_last_run
742 self.status = 0
743 self.runs = 0
744 self.result = []
745
746 def startTest(self, test):
747 test.imgs = getattr(test, "imgs", [])
748 # TestResult.startTest(self, test)
749 # just one buffer for both stdout and stderr
750 self.outputBuffer = io.StringIO()
751 stdout_redirector.fp = self.outputBuffer
752 stderr_redirector.fp = self.outputBuffer
753 self.stdout0 = sys.stdout
754 self.stderr0 = sys.stderr
755 sys.stdout = stdout_redirector
756 sys.stderr = stderr_redirector
757
758 def complete_output(self):
759 """
760 Disconnect output redirection and return buffer.
761 Safe to call multiple times.
762 """
763 if self.stdout0:
764 sys.stdout = self.stdout0
765 sys.stderr = self.stderr0
766 self.stdout0 = None
767 self.stderr0 = None
768 return self.outputBuffer.getvalue()
769
770 def stopTest(self, test):
771 # Usually one of addSuccess, addError or addFailure would have been called.
772 # But there are some path in unittest that would bypass this.
773 # We must disconnect stdout in stopTest(), which is guaranteed to be called.
774 if self.rerun and self.rerun >= 1:
775 if self.status == 1:
776 self.runs += 1
777 if self.runs <= self.rerun: 778 if self.save_last_run: 779 t = self.result.pop(-1) 780 if t[0] == 1: 781 self.failure_count -= 1 782 else: 783 self.error_count -= 1 784 test = copy.copy(test) 785 sys.stderr.write("Retesting… ") 786 sys.stderr.write(str(test)) 787 sys.stderr.write('..%d \n' % self.runs) 788 doc = getattr(test, '_testMethodDoc', u"") or u'' 789 if doc.find('->rerun') != -1:
790 doc = doc[:doc.find('->rerun')]
791 desc = "%s->rerun:%d" % (doc, self.runs)
792 if isinstance(desc, str):
793 desc = desc
794 test._testMethodDoc = desc
795 test(self)
796 else:
797 self.status = 0
798 self.runs = 0
799 self.complete_output()
800
801 def addSuccess(self, test):
802 self.success_count += 1
803 self.status = 0
804 TestResult.addSuccess(self, test)
805 output = self.complete_output()
806 self.result.append((0, test, output, ''))
807 if self.verbosity > 1:
808 sys.stderr.write('ok ')
809 sys.stderr.write(str(test))
810 sys.stderr.write('\n')
811 else:
812 sys.stderr.write('.' + str(self.success_count))
813
814 def addError(self, test, err):
815 self.error_count += 1
816 self.status = 1
817 TestResult.addError(self, test, err)
818 _, _exc_str = self.errors[-1]
819 output = self.complete_output()
820 self.result.append((2, test, output, _exc_str))
821 if not getattr(test, "driver", ""):
822 pass
823 else:
824 try:
825 driver = getattr(test, "driver")
826 test.imgs.append(driver.get_screenshot_as_base64())
827 except BaseException:
828 pass
829 if self.verbosity > 1:
830 sys.stderr.write('E ')
831 sys.stderr.write(str(test))
832 sys.stderr.write('\n')
833 else:
834 sys.stderr.write('E')
835
836 def addFailure(self, test, err):
837 self.failure_count += 1
838 self.status = 1
839 TestResult.addFailure(self, test, err)
840 _, _exc_str = self.failures[-1]
841 output = self.complete_output()
842 self.result.append((1, test, output, _exc_str))
843 if not getattr(test, "driver", ""):
844 pass
845 else:
846 try:
847 driver = getattr(test, "driver")
848 test.imgs.append(driver.get_screenshot_as_base64())
849 except BaseException:
850 pass
851 if self.verbosity > 1:
852 sys.stderr.write('F ')
853 sys.stderr.write(str(test))
854 sys.stderr.write('\n')
855 else:
856 sys.stderr.write('F')
857
858 def addSkip(self, test, reason):
859 self.skip_count += 1
860 self.status = 0
861 TestResult.addSkip(self, test, reason)
862 output = self.complete_output()
863 self.result.append((3, test, output, reason))
864 if self.verbosity > 1:
865 sys.stderr.write('S')
866 sys.stderr.write(str(test))
867 sys.stderr.write('\n')
868 else:
869 sys.stderr.write('S')
870
871
872 class HTMLTestRunner(Template_mixin):
873 """
874 """
875
876 def __init__(self, stream=sys.stdout, verbosity=1, title=None, description=None, save_last_run=True):
877 self.stream = stream
878 self.verbosity = verbosity
879 self.save_last_run = save_last_run
880 self.run_times = 0
881 if title is None:
882 self.title = self.DEFAULT_TITLE
883 else:
884 self.title = title
885 if description is None:
886 self.description = self.DEFAULT_DESCRIPTION
887 else:
888 self.description = description
889
890 self.startTime = datetime.datetime.now()
891
892 def run(self, test, rerun=0, save_last_run=False):
893 """Run the given test case or test suite."""
894 result = _TestResult(self.verbosity, rerun=rerun, save_last_run=save_last_run)
895 test(result)
896 self.stopTime = datetime.datetime.now()
897 self.run_times += 1
898 self.generateReport(test, result)
899 return result
900
901 def sortResult(self, result_list):
902 # unittest does not seems to run in any particular order.
903 # Here at least we want to group them together by class.
904 rmap = {}
905 classes = []
906 for n, t, o, e in result_list:
907 cls = t.__class__
908 if not cls in rmap:
909 rmap[cls] = []
910 classes.append(cls)
911 rmap[cls].append((n, t, o, e))
912 r = [(cls, rmap[cls]) for cls in classes]
913 return r
914
915 def getReportAttributes(self, result):
916 """
917 Return report attributes as a list of (name, value).
918 Override this to add custom attributes.
919 """
920 startTime = str(self.startTime)[:19]
921 duration = str(self.stopTime - self.startTime)
922 status = []
923 if result.success_count:
924 status.append('Passed:%s' % result.success_count)
925 if result.failure_count:
926 status.append('Failed:%s' % result.failure_count)
927 if result.error_count:
928 status.append('Errors:%s' % result.error_count)
929 if result.skip_count:
930 status.append('Skiped:%s' % result.skip_count)
931 if status:
932 status = ' '.join(status)
933 else:
934 status = 'none'
935 result = {
936 "pass": result.success_count,
937 "fail": result.failure_count,
938 "error": result.error_count,
939 "skip": result.skip_count,
940 }
941 return [
942 ('Start Time', startTime),
943 ('Duration', duration),
944 ('Status', status),
945 ('Result', result),
946 ]
947
948 def generateReport(self, test, result):
949 report_attrs = self.getReportAttributes(result)
950 generator = 'HTMLTestRunner %s' % __version__
951 stylesheet = self._generate_stylesheet()
952 heading = self._generate_heading(report_attrs)
953 report = self._generate_report(result)
954 ending = self._generate_ending()
955 chart = self._generate_chart(result)
956 output = self.HTML_TMPL % dict(
957 title=saxutils.escape(self.title),
958 generator=generator,
959 stylesheet=stylesheet,
960 heading=heading,
961 report=report,
962 ending=ending,
963 chart_script=chart,
964 channel=self.run_times,
965 )
966 self.stream.write(output.encode('utf8'))
967
968 def _generate_stylesheet(self):
969 return self.STYLESHEET_TMPL
970
971 def _generate_heading(self, report_attrs):
972 a_lines = []
973 for name, value in report_attrs:
974 result = {}
975 if name == "Result":
976 result = value
977 else:
978 line = self.HEADING_ATTRIBUTE_TMPL % dict(
979 name=saxutils.escape(name),
980 value=saxutils.escape(value),
981 )
982 a_lines.append(line)
983 heading = self.HEADING_TMPL % dict(
984 title=saxutils.escape(self.title),
985 parameters=''.join(a_lines),
986 description=saxutils.escape(self.description),
987 pass_count=saxutils.escape(str(result["pass"])),
988 fail_count=saxutils.escape(str(result["fail"])),
989 error_count=saxutils.escape(str(result["error"])),
990 skip_count=saxutils.escape(str(result["skip"])),
991 )
992 return heading
993
994 def _generate_report(self, result):
995 rows = []
996 sortedResult = self.sortResult(result.result)
997 for cid, (cls, cls_results) in enumerate(sortedResult):
998 # subtotal for a class
999 np = nf = ne = ns = 0
1000 for n, t, o, e in cls_results:
1001 if n == 0:
1002 np += 1
1003 elif n == 1:
1004 nf += 1
1005 elif n == 2:
1006 ne += 1
1007 else:
1008 ns += 1
1009
1010 # format class description
1011 if cls.__module__ == "__main__":
1012 name = cls.__name__
1013 else:
1014 name = "%s.%s" % (cls.__module__, cls.__name__)
1015 doc = cls.__doc__ or ""
1016 desc = doc and '%s: %s' % (name, doc) or name
1017
1018 row = self.REPORT_CLASS_TMPL % dict(
1019 style=ne > 0 and 'errorClass' or nf > 0 and 'failClass' or 'passClass',
1020 desc=desc,
1021 count=np + nf + ne,
1022 Pass=np,
1023 fail=nf,
1024 error=ne,
1025 cid='c%s.%s' % (self.run_times, cid + 1),
1026 )
1027 rows.append(row)
1028
1029 for tid, (n, t, o, e) in enumerate(cls_results):
1030 print("o", o)
1031 self._generate_report_test(rows, cid, tid, n, t, o, e)
1032
1033 report = self.REPORT_TMPL % dict(
1034 test_list=''.join(rows),
1035 count=str(result.success_count + result.failure_count + result.error_count),
1036 Pass=str(result.success_count),
1037 fail=str(result.failure_count),
1038 error=str(result.error_count),
1039 skip=str(result.skip_count),
1040 total=str(result.success_count + result.failure_count + result.error_count),
1041 channel=str(self.run_times),
1042 )
1043 return report
1044
1045 def _generate_chart(self, result):
1046 chart = self.ECHARTS_SCRIPT % dict(
1047 Pass=str(result.success_count),
1048 fail=str(result.failure_count),
1049 error=str(result.error_count),
1050 skip=str(result.skip_count),
1051 )
1052 return chart
1053
1054 def _generate_report_test(self, rows, cid, tid, n, t, o, e):
1055 # e.g. 'pt1.1', 'ft1.1','et1.1', 'st1.1' etc
1056 has_output = bool(o or e)
1057 if n == 0:
1058 tmp = "p"
1059 elif n == 1:
1060 tmp = "f"
1061 elif n == 2:
1062 tmp = "e"
1063 else:
1064 tmp = "s"
1065 tid = tmp + 't%d.%d.%d' % (self.run_times, cid + 1, tid + 1)
1066 # tid = (n == 0 and 'p' or 'f') + 't%s.%s' % (cid + 1, tid + 1)
1067 name = t.id().split('.')[-1]
1068 doc = t.shortDescription() or ""
1069 desc = doc and ('%s: %s' % (name, doc)) or name
1070 tmpl = has_output and self.REPORT_TEST_WITH_OUTPUT_TMPL or self.REPORT_TEST_NO_OUTPUT_TMPL
1071
1072 # o and e should be byte string because they are collected from stdout and stderr?
1073 if isinstance(o, str):
1074 # TODO: some problem with 'string_escape': it escape \n and mess up formating
1075 # uo = unicode(o.encode('string_escape'))
1076 uo = o
1077 else:
1078 uo = o
1079 if isinstance(e, str):
1080 # TODO: some problem with 'string_escape': it escape \n and mess up formating
1081 # ue = unicode(e.encode('string_escape'))
1082 ue = e
1083 else:
1084 ue = e
1085
1086 script = self.REPORT_TEST_OUTPUT_TMPL % dict(
1087 id=tid,
1088 output=saxutils.escape(uo + ue),
1089 )
1090 if getattr(t, 'imgs', []):
1091 # 判断截图列表，如果有则追加
1092 tmp = ""
1093 for i, img in enumerate(t.imgs):
1094 if i == 0:
1095 tmp += """\n""".format(img)
1096 else:
1097 tmp += """\n""".format(img)
1098 screenshots_html = self.IMG_TMPL.format(imgs=tmp)
1099 else:
1100 screenshots_html = """"""
1101
1102 row = tmpl % dict(
1103 tid=tid,
1104 Class=(n == 0 and 'hiddenRow' or 'none'),
1105 style=n == 2 and 'errorCase' or (n == 1 and 'failCase' or 'passCase'),
1106 desc=desc,
1107 script=script,
1108 status=self.STATUS[n],
1109 img=screenshots_html
1110 )
1111 rows.append(row)
1112 if not has_output:
1113 return
1114
1115 def _generate_ending(self):
1116 return self.ENDING_TMPL
1117
1118
1119 ##############################################################################
1120 # Facilities for running tests from the command line
1121 ##############################################################################
1122
1123 # Note: Reuse unittest.TestProgram to launch test. In the future we may
1124 # build our own launcher to support more specific command line
1125 # parameters like test title, CSS, etc.
1126 class TestProgram(unittest.TestProgram):
1127 """
1128 A variation of the unittest.TestProgram. Please refer to the base
1129 class for command line parameters.
1130 """
1131
1132 def runTests(self):
1133 # Pick HTMLTestRunner as the default test runner.
1134 # base class's testRunner parameter is not useful because it means
1135 # we have to instantiate HTMLTestRunner before we know self.verbosity.
1136 if self.testRunner is None:
1137 self.testRunner = HTMLTestRunner(verbosity=self.verbosity)
1138 unittest.TestProgram.runTests(self)
1139
1140
1141 main = TestProgram
1142
1143 ##############################################################################
1144 # Executing this module from the command line
1145 ##############################################################################
1146
1147 if __name__ == "__main__":
1148 main(module=None)