[TOOLS] Add a python script for syncing wine patches

This requires the pygit2 module
usage : ./winesync.py module <wine-tag> <winestaging-tag>

for instance ./winesync.py d3dx9 wine-4.1 v4.1

This requires to have a wine git checkout and a wine-staging checkout
configuration is done through yaml file named <module>.cfg specifying
the following:
 - file mappings
 - directory mappings
 - latest wine version the module is synced with
it then creates a local branch in the wine checkout, based on the given
tag, and then the staging script is ran on top of it.
Thanks to the mappings defined in the module configuration file, it then
create individual commits in the reactos git checkout with reworked
path.
In case of problem, it stops and lets you amend the latest commit and go
along with the process once this is done. (Makefile.in modified, new or
removed files, patches not cleanly applied)

Staging patches are added into the <module>_staging directory for the
ease of reverting them later. (TODO)

See previous [WINESYNC] commits to have an overview of the result

.gitignore

@@ -11,3 +11,4 @@ modules/AHK_Tests
 .settings
 build
 .vscode
+sdk/tools/winesync/winesync.cfg

sdk/tools/winesync/winesync.py

@@ -0,0 +1,354 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import string
+import argparse
+import subprocess
+import fnmatch
+import pygit2
+import yaml
+
+def string_to_valid_file_name(to_convert):
+    valid_chars = '-_.()' + string.ascii_letters + string.digits
+    result = ''
+    for c in to_convert:
+        if c in valid_chars:
+            result += c
+        else:
+            result += '_'
+    # strip final dot, if any
+    if result.endswith('.'):
+        return result[:-1]
+    return result
+
+class wine_sync:
+    def __init__(self, module):
+        if os.path.isfile('winesync.cfg'):
+            with open('winesync.cfg', 'r') as file_input:
+                config = yaml.load(file_input, Loader=yaml.FullLoader)
+            self.reactos_src = config['repos']['reactos']
+            self.wine_src = config['repos']['wine']
+            self.wine_staging_src = config['repos']['wine-staging']
+        else:
+            config = { }
+            self.reactos_src = input('Please enter the path to the reactos git tree: ')
+            self.wine_src = input('Please enter the path to the wine git tree: ')
+            self.wine_staging_src = input('Please enter the path to the wine-staging git tree: ')
+            config['repos'] = { 'reactos' : self.reactos_src,
+                                'wine': self.wine_src,
+                                'wine-staging': self.wine_staging_src }
+            with open('winesync.cfg', 'w') as file_output:
+                yaml.dump(config, file_output)
+
+        self.wine_repo = pygit2.Repository(self.wine_src)
+        self.wine_staging_repo = pygit2.Repository(self.wine_staging_src)
+        self.reactos_repo = pygit2.Repository(self.reactos_src)
+
+        # read the index from the reactos tree
+        self.reactos_index = self.reactos_repo.index
+        self.reactos_index.read()
+
+        # get the actual state for the asked module
+        self.module = module
+        with open(module + '.cfg', 'r') as file_input:
+            self.module_cfg = yaml.load(file_input, Loader=yaml.FullLoader)
+
+        self.staged_patch_dir = os.path.join('sdk', 'tools', 'winesync', self.module + '_staging')
+
+    def create_or_checkout_wine_branch(self, wine_tag, wine_staging_tag):
+        wine_branch_name = 'winesync-' + wine_tag + '-' + wine_staging_tag
+        branch = self.wine_repo.lookup_branch(wine_branch_name)
+        if branch is None:
+            # get our target commits
+            wine_target_commit = self.wine_repo.revparse_single(wine_tag)
+            if isinstance(wine_target_commit, pygit2.Tag):
+                wine_target_commit = wine_target_commit.target
+
+            wine_staging_target_commit = self.wine_staging_repo.revparse_single(wine_staging_tag)
+            if isinstance(wine_staging_target_commit, pygit2.Tag):
+                wine_staging_target_commit = wine_staging_target_commit.target
+
+            self.wine_repo.branches.local.create(wine_branch_name, self.wine_repo.revparse_single('HEAD'))
+            self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
+            self.wine_repo.reset(wine_target_commit, pygit2.GIT_RESET_HARD)
+
+            # do the same for the wine-staging tree
+            self.wine_staging_repo.branches.local.create(wine_branch_name, self.wine_staging_repo.revparse_single('HEAD'))
+            self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch(wine_branch_name))
+            self.wine_staging_repo.reset(wine_staging_target_commit, pygit2.GIT_RESET_HARD)
+
+            # run the wine-staging script
+            subprocess.call(['bash', '-c', self.wine_staging_src + '/patches/patchinstall.sh DESTDIR=' + self.wine_src + ' --all --backend=git-am'])
+
+            # delete the branch we created
+            self.wine_staging_repo.checkout(self.wine_staging_repo.lookup_branch('master'))
+            self.wine_staging_repo.branches.delete(wine_branch_name)
+        else:
+            self.wine_repo.checkout(self.wine_repo.lookup_branch(wine_branch_name))
+
+        return wine_branch_name
+
+    # helper function for resolving wine tree path to reactos one
+    # Note : it doesn't care about the fact that the file actually exists or not
+    def wine_to_reactos_path(self, wine_path):
+        if wine_path in self.module_cfg['files']:
+            # we have a direct mapping
+            return self.module_cfg['files'][wine_path]
+
+        if not '/' in wine_path:
+            # root files should have a direct mapping
+            return None
+
+        wine_dir, wine_file = os.path.split(wine_path)
+        if wine_dir in self.module_cfg['directories']:
+            # we have a mapping for the directory
+            return os.path.join(self.module_cfg['directories'][wine_dir], wine_file)
+
+        # no match
+        return None
+
+    def sync_wine_commit(self, wine_commit, in_staging, staging_patch_index):
+        # Get the diff object
+        diff = self.wine_repo.diff(wine_commit.parents[0], wine_commit)
+
+        modified_files = []
+        ignored_files = []
+        warning_message = ''
+        complete_patch = ''
+
+        if in_staging:
+            # see if we already applied this patch
+            patch_file_name = f'{staging_patch_index:04}-{string_to_valid_file_name(wine_commit.message.splitlines()[0])}.diff'
+            patch_path = os.path.join(self.reactos_src, self.staged_patch_dir, patch_file_name)
+            if os.path.isfile(patch_path):
+                print(f'Skipping patch as {patch_path} already exists')
+                return True, ''
+
+        for delta in diff.deltas:
+            if delta.old_file.path == '/dev/null':
+                # check if we should care
+                new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
+                if not new_reactos_path is None:
+                    warning_message += 'file ' + delta.new_file.path + ' is added to the wine tree !\n'
+                    old_reactos_path = '/dev/null'
+                else:
+                    old_reactos_path = None
+            elif delta.new_file.path == '/dev/null':
+                # check if we should care
+                old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
+                if not old_reactos_path is None:
+                    warning_message += 'file ' + delta.old_file.path + ' is removed from the wine tree !\n'
+                    new_reactos_path = '/dev/null'
+                else:
+                    new_reactos_path = None
+            elif delta.new_file.path.endswith('Makefile.in'):
+                warning_message += 'file ' + delta.new_file.path + ' was modified !\n'
+                # no need to warn that those are ignored, we just did.
+                continue
+            else:
+                new_reactos_path = self.wine_to_reactos_path(delta.new_file.path)
+                old_reactos_path = self.wine_to_reactos_path(delta.old_file.path)
+
+            if (new_reactos_path is not None) or (old_reactos_path is not None):
+                # print('Must apply diff: ' + old_reactos_path + ' --> ' + new_reactos_path)
+                new_blob = self.wine_repo.get(wine_commit.tree[delta.new_file.path].id)
+                old_blob = self.wine_repo.get(wine_commit.parents[0].tree[delta.old_file.path].id)
+
+                blob_patch = pygit2.Patch.create_from(
+                    old=old_blob,
+                    new=new_blob,
+                    old_as_path=old_reactos_path,
+                    new_as_path=new_reactos_path)
+
+                # print(str(wine_commit.id))
+                # print(blob_patch.text)
+
+                # this doesn't work
+                # reactos_diff = pygit2.Diff.parse_diff(blob_patch.text)
+                # reactos_repo.apply(reactos_diff)
+                try:
+                    subprocess.run(['git', '-C', self.reactos_src, 'apply', '--reject'], input=blob_patch.data, check=True)
+                except subprocess.CalledProcessError as err:
+                    warning_message += 'Error while applying patch to ' + new_reactos_path + '\n'
+                self.reactos_index.add(new_reactos_path)
+
+                complete_patch += blob_patch.text
+
+                modified_files += [delta.old_file.path, delta.new_file.path]
+            else:
+                ignored_files += [delta.old_file.path, delta.new_file.path]
+
+        if not modified_files:
+            # We applied nothing
+            return False, ''
+
+        print('Applied patches from wine commit ' + str(wine_commit.id))
+
+        if ignored_files:
+            warning_message += 'WARNING : some files were ignored: ' + ' '.join(ignored_files) + '\n'
+
+        if not in_staging:
+            self.module_cfg['tags']['wine'] = str(wine_commit.id)
+            with open(self.module + '.cfg', 'w') as file_output:
+                yaml.dump(self.module_cfg, file_output)
+            self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
+        else:
+            # Add the staging patch
+            # do not save the wine commit ID in <module>.cfg, as it's a local one for staging patches
+            if not os.path.isdir(os.path.join(self.reactos_src, self.staged_patch_dir)):
+                os.mkdir(os.path.join(self.reactos_src, self.staged_patch_dir))
+            with open(patch_path, 'w') as file_output:
+                file_output.write(complete_patch)
+            self.reactos_index.add(os.path.join(self.staged_patch_dir, patch_file_name))
+
+        self.reactos_index.write()
+
+        commit_msg = f'[WINESYNC] {wine_commit.message}\n'
+        if (in_staging):
+            commit_msg += f'wine-staging patch by {wine_commit.author.name} <{wine_commit.author.email}>'
+        else:
+            commit_msg += f'wine commit id {str(wine_commit.id)} by {wine_commit.author.name} <{wine_commit.author.email}>'
+
+        self.reactos_repo.create_commit('HEAD',
+            pygit2.Signature('winesync', 'ros-dev@reactos.org'),
+            self.reactos_repo.default_signature,
+            commit_msg,
+            self.reactos_index.write_tree(),
+            [self.reactos_repo.head.target])
+
+        if (warning_message != ''):
+            warning_message += 'If needed, amend the current commit in your reactos tree and start this script again'
+
+            if not in_staging:
+                warning_message += f'You can see the details of the wine commit here: https://source.winehq.org/git/wine.git/commit/{str(wine_commit.id)}'
+            else:
+                warning_message += 'Do not forget to run\n'
+                warning_message += f'git diff HEAD^ \':(exclude)sdk/tools/winesync/{patch_file_name}\' > sdk/tools/winesync/{patch_file_name}\n'
+                warning_message += 'after your correction and then\n'
+                warning_message += f'git add sdk/tools/winesync/{patch_file_name}\n'
+                warning_message += 'before running "git commit --amend"'
+
+        return True, warning_message
+
+    def revert_staged_patchset(self):
+        # revert all of this in one commmit
+        staged_patch_dir_path = os.path.join(self.reactos_src, self.staged_patch_dir)
+        if not os.path.isdir(staged_patch_dir_path):
+            return True
+
+        has_patches = False
+
+        for patch_file_name in sorted(os.listdir(staged_patch_dir_path), reverse=True):
+            patch_path = os.path.join(staged_patch_dir_path, patch_file_name)
+            if not os.path.isfile(patch_path):
+                continue
+
+            has_patches = True
+
+            with open(patch_path, 'rb') as patch_file:
+                try:
+                    subprocess.run(['git', '-C', self.reactos_src, 'apply', '-R', '--reject'], stdin=patch_file, check=True)
+                except subprocess.CalledProcessError as err:
+                    print(f'Error while reverting patch {patch_file_name}')
+                    print('Please check, remove the offending patch with git rm, and relaunch this script')
+                    return False
+
+            self.reactos_index.remove(os.path.join(self.staged_patch_dir, patch_file_name))
+            self.reactos_index.write()
+            os.remove(patch_path)
+
+        if not has_patches:
+            return True
+
+        self.reactos_index.add_all([f for f in self.module_cfg['files'].values()])
+        self.reactos_index.add_all([f'{d}/*.*' for d in self.module_cfg['directories'].values()])
+        self.reactos_index.write()
+
+        self.reactos_repo.create_commit(
+            'HEAD',
+            self.reactos_repo.default_signature,
+            self.reactos_repo.default_signature,
+            f'[WINESYNC]: revert wine-staging patchset for {self.module}',
+            self.reactos_index.write_tree(),
+            [self.reactos_repo.head.target])
+        return True
+
+    def sync_to_wine(self, wine_tag, wine_staging_tag):
+        # Get our target commit
+        wine_target_commit = self.wine_repo.revparse_single(wine_tag)
+        if isinstance(wine_target_commit, pygit2.Tag):
+            wine_target_commit = wine_target_commit.target
+        # print(f'wine target commit is {wine_target_commit}')
+
+        # get the wine commit id where we left
+        in_staging = False
+        wine_last_sync = self.wine_repo.revparse_single(self.module_cfg['tags']['wine'])
+        if (isinstance(wine_last_sync, pygit2.Tag)):
+            if not self.revert_staged_patchset():
+                return
+            wine_last_sync = wine_last_sync.target
+        if (isinstance(wine_last_sync, pygit2.Commit)):
+            wine_last_sync = wine_last_sync.id
+
+        # create a branch to keep things clean
+        wine_branch_name = self.create_or_checkout_wine_branch(wine_tag, wine_staging_tag)
+
+        finished_sync = True
+        staging_patch_index = 1
+
+        # walk each commit between last sync and the asked tag/revision
+        wine_commit_walker = self.wine_repo.walk(self.wine_repo.head.target, pygit2.GIT_SORT_TOPOLOGICAL | pygit2.GIT_SORT_REVERSE)
+        wine_commit_walker.hide(wine_last_sync)
+        for wine_commit in wine_commit_walker:
+            applied_patch, warning_message = self.sync_wine_commit(wine_commit, in_staging, staging_patch_index)
+
+            if str(wine_commit.id) == str(wine_target_commit):
+                print('We are now in staging territory')
+                in_staging = True
+
+            if not applied_patch:
+                continue
+
+            if in_staging:
+                staging_patch_index += 1
+
+            if warning_message != '':
+                print(warning_message)
+                finished_sync = False
+                break
+
+        # we're done without error
+        if finished_sync:
+            # update wine tag and commit
+            self.module_cfg['tags']['wine'] = wine_tag
+            with open(self.module + '.cfg', 'w') as file_output:
+                yaml.dump(self.module_cfg, file_output)
+
+            self.reactos_index.add(f'sdk/tools/winesync/{self.module}.cfg')
+            self.reactos_index.write()
+            self.reactos_repo.create_commit(
+                'HEAD',
+                self.reactos_repo.default_signature,
+                self.reactos_repo.default_signature,
+                f'[WINESYNC]: {self.module} is now in sync with wine-staging {wine_tag}',
+                self.reactos_index.write_tree(),
+                [self.reactos_repo.head.target])
+
+            print('The branch ' + wine_branch_name + ' was created in your wine repository. You might want to delete it, but you should keep it in case you want to sync more module up to this wine version')
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('module', help='The module you want to sync. <module>.cfg must exist in the current directory')
+    parser.add_argument('wine_tag', help='The wine tag or commit id to sync to')
+    parser.add_argument('wine_staging_tag', help='The wine staging tag or commit id to pick wine staged patches from')
+
+    args = parser.parse_args()
+
+    syncator = wine_sync(args.module)
+
+    return syncator.sync_to_wine(args.wine_tag, args.wine_staging_tag)
+
+
+if __name__ == '__main__':
+    main()