From 9a27bc87984613a63cb6cfb651282b824df51c81 Mon Sep 17 00:00:00 2001 From: Edwin Mons Date: Mon, 21 Jan 2019 10:09:48 +0100 Subject: Make Version.py compatible with both Py2 and Py3 The merge of 4.x changes onto master introduced changes to Pyhon scripts that weren't compatible with Python 3. The Version.py script is now compatible with both Python 2 and Python 3, and a slightly underimplemented test has been expanded to actually live up to its name. getGitBuildVersion has been refactored completely. It has been simplified to require only one invocation of git describe, unit tests for the parsing have been added, and a way to call getGitBuildVersion from the commandline has been added to Version.py (to avoid adding a unit test that would call out to an external tool). DocBook.py has been made compatible with Python3, and some additional logic to prevent emitting b'path/to/docbook/xml' instead of the desired 'path/to/docbook/xml' has been added. Generation of COPYING in Swift/QtUI now uses the upstream version provided with SCons 3, and our custom version has been removed. Unused code in the SwiftenDevelopersGuide SConscript with invalid regular expressions has been removed, and the remaining regular expressions in SConscripts have been fixed. Test-Information Code has been tested on macOS 10.14. Scons now completes a build on a clean tree using either Python 2 or Python 3. Running it on a previouly built tree works with either, as well, mixing versions between invocations is not an issue. Swift unit tests pass with Python 3.7.0. Version.py unit tests pass with Python 2.7.15 and 3.7.0. Running with doc=1 works with both Python 2 and Python 3, even if the docbook points to a path with unicode characters in it. Resulting COPYING file has been verified both visually and against a Python2 generated one on master. Resulting XML files for documentation have been inspected. Resulting manual HTML and PDF files have been inspected. Change-Id: I54de909d80b8e35a8c351261ae10ce3537729c84 diff --git a/BuildTools/DocBook/SCons/DocBook.py b/BuildTools/DocBook/SCons/DocBook.py index ffb0bfc..d7c95ba 100644 --- a/BuildTools/DocBook/SCons/DocBook.py +++ b/BuildTools/DocBook/SCons/DocBook.py @@ -5,6 +5,19 @@ import SCons.Util, SCons.Action import xml.dom.minidom, re, os.path, sys +def maybeBytesToString(s): + if isinstance(s, bytes): + return s.decode('utf-8') + return s + +def prepareForWrite(s): + try: + if isinstance(s, unicode): + return s.encode('utf-8') + except NameError: + pass + return s + def generate(env) : # Location of stylesheets and catalogs docbook_dir = "#/BuildTools/DocBook" @@ -32,15 +45,15 @@ def generate(env) : rewritePrefix="%(docbook_xsl_dir)s/" /> """ - docbook_xml_dir = source[0].get_contents() - docbook_xsl_dir = source[1].get_contents() + docbook_xml_dir = maybeBytesToString(source[0].get_contents()) + docbook_xsl_dir = maybeBytesToString(source[1].get_contents()) if env["PLATFORM"] == "win32" : docbook_xml_dir = docbook_xml_dir.replace("\\","/") docbook_xsl_dir = docbook_xsl_dir.replace("\\","/") file = open(target[0].abspath, "w") file.write(catalog % { - "docbook_xml_dir" : docbook_xml_dir, - "docbook_xsl_dir" : docbook_xsl_dir, + "docbook_xml_dir" : prepareForWrite(docbook_xml_dir), + "docbook_xsl_dir" : prepareForWrite(docbook_xsl_dir), }) file.close() diff --git a/BuildTools/SCons/Tools/textfile.py b/BuildTools/SCons/Tools/textfile.py deleted file mode 100644 index 73105ad..0000000 --- a/BuildTools/SCons/Tools/textfile.py +++ /dev/null @@ -1,179 +0,0 @@ -# -*- python -*- -# -# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 The SCons Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY -# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -# - -__doc__ = """ -Textfile/Substfile builder for SCons. - - Create file 'target' which typically is a textfile. The 'source' - may be any combination of strings, Nodes, or lists of same. A - 'linesep' will be put between any part written and defaults to - os.linesep. - - The only difference between the Textfile builder and the Substfile - builder is that strings are converted to Value() nodes for the - former and File() nodes for the latter. To insert files in the - former or strings in the latter, wrap them in a File() or Value(), - respectively. - - The values of SUBST_DICT first have any construction variables - expanded (its keys are not expanded). If a value of SUBST_DICT is - a python callable function, it is called and the result is expanded - as the value. Values are substituted in a "random" order; if any - substitution could be further expanded by another subsitition, it - is unpredictible whether the expansion will occur. -""" - -__revision__ = "src/engine/SCons/Tool/textfile.py 5357 2011/09/09 21:31:03 bdeegan" - -import SCons - -import os -import re - -from SCons.Node import Node -from SCons.Node.Python import Value -from SCons.Util import is_String, is_Sequence, is_Dict - -def _do_subst(node, subs): - """ - Fetch the node contents and replace all instances of the keys with - their values. For example, if subs is - {'%VERSION%': '1.2345', '%BASE%': 'MyProg', '%prefix%': '/bin'}, - then all instances of %VERSION% in the file will be replaced with - 1.2345 and so forth. - """ - contents = node.get_contents().decode('utf-8') - if not subs: return contents - for (k,v) in subs: - contents = re.sub(k, v, contents) - return contents - -def _action(target, source, env): - # prepare the line separator - linesep = env['LINESEPARATOR'] - if linesep is None: - linesep = os.linesep - elif is_String(linesep): - pass - elif isinstance(linesep, Value): - linesep = linesep.get_text_contents() - else: - raise SCons.Errors.UserError( - 'unexpected type/class for LINESEPARATOR: %s' - % repr(linesep), None) - - # create a dictionary to use for the substitutions - if 'SUBST_DICT' not in env: - subs = None # no substitutions - else: - d = env['SUBST_DICT'] - if is_Dict(d): - d = list(d.items()) - elif is_Sequence(d): - pass - else: - raise SCons.Errors.UserError('SUBST_DICT must be dict or sequence') - subs = [] - for (k,v) in d: - if callable(v): - v = v() - if is_String(v): - v = env.subst(v) - else: - v = str(v) - subs.append((k,v)) - - # write the file - try: - fd = open(target[0].get_path(), "wb") - except (OSError,IOError) as e: - raise SCons.Errors.UserError("Can't write target file %s" % target[0]) - # separate lines by 'linesep' only if linesep is not empty - lsep = None - for s in source: - if lsep: fd.write(lsep) - stringtowrite = _do_subst(s, subs) - if isinstance(stringtowrite, str): - fd.write(stringtowrite) - elif isinstance(stringtowrite, unicode): - fd.write(stringtowrite.encode('utf-8')) - lsep = linesep - fd.close() - -def _strfunc(target, source, env): - return "Creating '%s'" % target[0] - -def _convert_list_R(newlist, sources): - for elem in sources: - if is_Sequence(elem): - _convert_list_R(newlist, elem) - elif isinstance(elem, Node): - newlist.append(elem) - else: - newlist.append(Value(elem)) -def _convert_list(target, source, env): - if len(target) != 1: - raise SCons.Errors.UserError("Only one target file allowed") - newlist = [] - _convert_list_R(newlist, source) - return target, newlist - -_common_varlist = ['SUBST_DICT', 'LINESEPARATOR'] - -_text_varlist = _common_varlist + ['TEXTFILEPREFIX', 'TEXTFILESUFFIX'] -_text_builder = SCons.Builder.Builder( - action = SCons.Action.Action(_action, _strfunc, varlist = _text_varlist), - source_factory = Value, - emitter = _convert_list, - prefix = '$TEXTFILEPREFIX', - suffix = '$TEXTFILESUFFIX', - ) - -_subst_varlist = _common_varlist + ['SUBSTFILEPREFIX', 'TEXTFILESUFFIX'] -_subst_builder = SCons.Builder.Builder( - action = SCons.Action.Action(_action, _strfunc, varlist = _subst_varlist), - source_factory = SCons.Node.FS.File, - emitter = _convert_list, - prefix = '$SUBSTFILEPREFIX', - suffix = '$SUBSTFILESUFFIX', - src_suffix = ['.in'], - ) - -def generate(env): - env['LINESEPARATOR'] = os.linesep - env['BUILDERS']['MyTextfile'] = _text_builder - env['TEXTFILEPREFIX'] = '' - env['TEXTFILESUFFIX'] = '.txt' - env['BUILDERS']['MySubstfile'] = _subst_builder - env['SUBSTFILEPREFIX'] = '' - env['SUBSTFILESUFFIX'] = '' - -def exists(env): - return 1 - -# Local Variables: -# tab-width:4 -# indent-tabs-mode:nil -# End: -# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/BuildTools/SCons/Version.py b/BuildTools/SCons/Version.py index 001374a..23cd850 100644 --- a/BuildTools/SCons/Version.py +++ b/BuildTools/SCons/Version.py @@ -1,26 +1,34 @@ +from __future__ import print_function import subprocess, os, datetime, re, os.path, sys, unittest -def getGitBuildVersion(root, project) : - tag = git("describe --tags --exact --match \"" + project + "-*\"", root) - if tag : - return tag.rstrip()[len(project)+1:] - tag = git("describe --tags --match \"" + project + "-*\"", root) - if tag : - m = re.match(project + "-(.*)-(.*)-(.*)", tag) - if m : - return m.group(1) + "-dev" + m.group(2) +def gitDescribeToVersion(tag, tagPrefix): + m = re.match(r'^' + tagPrefix + r'-(.+?)(?:-(.+?)-(.+?))?$', tag) + if not m: + return None + result = m.group(1) + if m.group(2): + result += "-dev" + m.group(2) + return result + +def getGitBuildVersion(root, tagPrefix) : + tag = git("describe --tags --match \"" + tagPrefix + "-*\"", root) + if tag: + return gitDescribeToVersion(tag, tagPrefix) return None -def git(cmd, root) : +def git(cmd, root): full_cmd = "git " + cmd + # Would've used with .. as p, but that's not supported by Python 2.7 p = subprocess.Popen(full_cmd, cwd=root, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=(os.name != "nt")) gitVersion = p.stdout.read() - # error = p.stderr.read() - # if error: - # print("Git error: " + error) - p.stdin.close() - if p.wait() == 0 : - return gitVersion + try: + p.stdin.close() + p.stdout.close() + p.stderr.close() + except: + pass + if p.wait() == 0: + return bytes(gitVersion).decode('utf-8') return None def getBuildVersion(root, project) : @@ -95,7 +103,7 @@ def convertToWindowsVersion(version): return (major, minor, patch) # Test Windows version mapping scheme -class convertToWindowsVersionTest(unittest.TestCase): +class TestCases(unittest.TestCase): def testWindowsVersionsAreDescending(self): versionStringsWithOldVersions = [ ("5.0rc11", None), @@ -111,13 +119,13 @@ class convertToWindowsVersionTest(unittest.TestCase): ("4.0rc5", (4, 0, 50000)), ("4.0rc4", (4, 0, 40000)), ("4.0rc3", (4, 0, 30000)), + ('4.0rc2-dev39', (4, 0, 20039)), ('4.0rc2-dev34', (4, 0, 20034)), ('4.0rc2-dev33', (4, 0, 20033)), ('4.0rc2-dev31', (4, 0, 20031)), ('4.0rc2-dev30', (4, 0, 20030)), ('4.0rc2-dev29', (4, 0, 20029)), ('4.0rc2-dev27', (4, 0, 20027)), - ('4.0rc2-dev39', (4, 0, 20039)), ('4.0rc2', (4, 0, 20000)), ('4.0rc1', (4, 0, 10000)), ('4.0beta2-dev203', (4, 0, 2203)), @@ -163,7 +171,17 @@ class convertToWindowsVersionTest(unittest.TestCase): ('3.0alpha-dev524', (3, 0, 524)), ('3.0alpha-dev515', (3, 0, 515)), ] - windowsVersionMapping = list(map(lambda (x,y): (x, convertToWindowsVersion(x)), versionStringsWithOldVersions)) + prevParsed = None + prevOldVersion = None + for string, oldVersion in versionStringsWithOldVersions: + parsed = convertToWindowsVersion(string) + if prevOldVersion and oldVersion: + self.assertTrue(oldVersion <= prevOldVersion, "Old version %r must come before %r" % (oldVersion, prevOldVersion)) + if prevParsed: + self.assertTrue(parsed < prevParsed, "%s => %r must be smaller than %s => %r" % (string, parsed, prevString, prevParsed)) + prevString = string + prevParsed = parsed + prevOldVersion = oldVersion def testThatBetaIsHigherThanAlpha(self): self.assertTrue(convertToWindowsVersion("3.0beta0") > convertToWindowsVersion("3.0alpha0")) @@ -190,12 +208,24 @@ class convertToWindowsVersionTest(unittest.TestCase): self.assertTrue(convertToWindowsVersion("3.0") > convertToWindowsVersion("3.0rc1")) self.assertTrue(convertToWindowsVersion("3.0") > convertToWindowsVersion("3.0rc11")) + def testGitDescribeToVersion(self): + self.assertEqual("5.0alpha2-dev100", gitDescribeToVersion("swift-5.0alpha2-100-g33d5f6571", "swift")) + self.assertEqual("5.0alpha2", gitDescribeToVersion("swift-5.0alpha2", "swift")) + self.assertEqual("5.0-dev1", gitDescribeToVersion("swift-5.0-1-g012312312", "swift")) + self.assertIsNone(gitDescribeToVersion("swiften-5.0-1-g012312312", "swift")) + self.assertIsNone(gitDescribeToVersion("quickly-5.0-1-g012312312", "swift")) + self.assertIsNone(gitDescribeToVersion("swift-", "swift")) + self.assertIsNone(gitDescribeToVersion("swift", "swift")) + if __name__ == '__main__': if len(sys.argv) == 1: unittest.main() elif len(sys.argv) == 2: - print convertToWindowsVersion(sys.argv[1]) + if sys.argv[1] == "--git-build-version": + print(getGitBuildVersion(os.path.dirname(os.path.dirname(__file__)), "swift")) + else: + print(convertToWindowsVersion(sys.argv[1])) sys.exit(0) else: - print "Error: Simply run the script without arguments or pass a single argument." + print("Error: Simply run the script without arguments or pass a single argument.") sys.exit(-1) diff --git a/Documentation/SwiftenDevelopersGuide/SConscript b/Documentation/SwiftenDevelopersGuide/SConscript index ac7c67a..95e5c87 100644 --- a/Documentation/SwiftenDevelopersGuide/SConscript +++ b/Documentation/SwiftenDevelopersGuide/SConscript @@ -9,36 +9,6 @@ env.Tool("DocBook", toolpath = ["#/BuildTools/DocBook/SCons"]) import sys, re, os.path def generateDocBookCode(env, target, source) : - # Strips empty lines from the beginning & end of a program - def stripEmptyLines(program) : - programLines = program.split('\n') - newProgramLines = [] - inProgram = False - for line in programLines : - if not re.match("^\s*$", line) or inProgram : - inProgram = True - newProgramLines.append(line) - return '\n'.join(newProgramLines).rstrip() - - def createCallouts(program, calloutPrefix) : - newProgramLines = [] - calloutLines = [] - nextID = 0 - for line in program.split("\n") : - # FIXME: Takes the largest match - m = re.match(".*\/* \(\*\) (.*) \*/.*", line) - if m : - cobID = "cob-" + calloutPrefix + "-" + str(nextID) - coID = "co-" + calloutPrefix + "-" + str(nextID) - nextID += 1 - line = re.sub("/\*.*\*/", "]]>%(text)s" % {"cobID": cobID, "coID": coID, "text": m.group(1)}) - newProgramLines.append(line) - callouts = "" - if len(calloutLines) > 0 : - callouts = "" + "\n".join(calloutLines) + "" - return ("\n".join(newProgramLines), callouts) - # Parse program filename = source[0].abspath filenameBase = os.path.basename(filename).replace(".cpp", "") @@ -71,10 +41,7 @@ def generateDocBookCode(env, target, source) : inputfile.close() for programName, program in programs.items() : - program = stripEmptyLines(program) - (program, callouts) = createCallouts(program, filenameBase + "-" + programName) - - document = "" + callouts + "" + document = "" # Generate code output = open(target[0].abspath, 'w') diff --git a/Swift/QtUI/SConscript b/Swift/QtUI/SConscript index a2ad9b1..584cfea 100644 --- a/Swift/QtUI/SConscript +++ b/Swift/QtUI/SConscript @@ -24,7 +24,7 @@ def generateQRCTheme(dir, prefix) : Import("env") -myenv = env.Clone() +myenv = env.Clone(tools = [ 'textfile' ]) # Disable warnings that affect Qt myenv["CXXFLAGS"] = list(filter(lambda x : x != "-Wfloat-equal", myenv["CXXFLAGS"])) @@ -365,7 +365,7 @@ myenv["TEXTFILESUFFIX"] = "" copying_files = [myenv.File("../../COPYING.gpl"), myenv.File("../../COPYING.thirdparty"), myenv.File("../../COPYING.dependencies")] if env["PLATFORM"] == "darwin" and env["HAVE_SPARKLE"] : copying_files.append(env["SPARKLE_COPYING"]) -myenv.MyTextfile(target = "COPYING", source = copying_files, LINESEPARATOR = "\n\n========\n\n\n") +myenv.Textfile(target = "COPYING", source = copying_files, LINESEPARATOR = "\n\n========\n\n\n") ################################################################################ # Translation diff --git a/Swiften/SConscript b/Swiften/SConscript index 4deddaf..5705113 100644 --- a/Swiften/SConscript +++ b/Swiften/SConscript @@ -12,7 +12,7 @@ external_swiften_dep_modules = ["BOOST"] if env["SCONS_STAGE"] == "flags" : env["SWIFTEN_DLL"] = env["swiften_dll"] env["SWIFTEN_VERSION"] = Version.getBuildVersion(env.Dir("#").abspath, "swift") - version_match = re.match("(\d+)\.(\d+).*", env["SWIFTEN_VERSION"]) + version_match = re.match(r"(\d+)\.(\d+).*", env["SWIFTEN_VERSION"]) if version_match : env["SWIFTEN_VERSION_MAJOR"] = int(version_match.group(1)) env["SWIFTEN_VERSION_MINOR"] = int(version_match.group(2)) -- cgit v0.10.2-6-g49f6