diff --git a/tools/deb2rpm/.gitignore b/tools/deb2rpm/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..368cfcf9880b0c090d28f148cc78a17378c07e23 --- /dev/null +++ b/tools/deb2rpm/.gitignore @@ -0,0 +1,7 @@ +# IDE +.idea + +database/deb2rpm.db +__pycache__ + + diff --git a/tools/deb2rpm/README.en.md b/tools/deb2rpm/README.en.md new file mode 100644 index 0000000000000000000000000000000000000000..13cf1e0dc587d1b65e12f37a302389b61e8b00e4 --- /dev/null +++ b/tools/deb2rpm/README.en.md @@ -0,0 +1,36 @@ +# deb2rpm + +#### Description +{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/tools/deb2rpm/README.md b/tools/deb2rpm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..feb14e549a8181cf9916b5b2b03a086f794a9a30 --- /dev/null +++ b/tools/deb2rpm/README.md @@ -0,0 +1,53 @@ +# deb2rpm + +#### 介绍 +本项目目标为openEuler引入Debian的软件包，将deb包转化为openEuler上可用的rpm包及srpm源码包。 + +#### 软件架构 +- 主要支持openEuler-22.03_LTS +- deb包源支持Ubuntu各版本，如jammy(20)、bionic(18)等 + +#### 安装教程 + +1. 在家目录下`git clone`本项目 +2. 执行`./initialize.sh`进行初始化(数据库初始化可能需要较长时间) + + +#### 使用说明 + +执行`python3 deb2rpm.py -h`查看帮助信息如下： +``` +usage: deb2rpm.py [-h] [-d DEP_JSON] [-D DEEP_DEP_JSON] [-t BUILD_TYPE] [-s] [-k] [-a] [-b] target + +Build the target from deb to rpm + +positional arguments: + target The target package + +options: + -h, --help show this help message and exit + -d DEP_JSON, --dep-json DEP_JSON + Specify the path of json file with target dependency + -D DEEP_DEP_JSON, --deep-dep-json DEEP_DEP_JSON +<<<<<<< HEAD +======= + Specify the path of json file with all recursive dependency of target +>>>>>>> c2ad216 (Update docs) + -t BUILD_TYPE, --build-type BUILD_TYPE + The type of building: source_and_binary(default) | source | binary + -s, --show-dep-only Only show the dependency infomation but not build + -k, --keep-all-rpms Use if you want to keep all the rpms, else only keep rpms of target + -a, --auto-build Auto build the target recursively + -b, --build-dep Install all the exist build-depends +``` +对以上参数的具体说明如下： +- target：需要转换的deb源码包名，比如`zip`、`nginx`等 +- `-d`、`--dep-json`：指定target的依赖关系json文件路径，允许相对路径和绝对路径，如果不指定则不保存对应的json文件 +- `-D`、 `--deep-dep-json`：指定target的所有递归全量依赖关系json文件路径，允许相对路径和绝对路径，如果不指定则不保存对应的json文件 +- `-t`、`--build-type`：指定构建类型，可选值为`source_and_binary`、`source`、`binary`，默认为`source_and_binary`，分别对应源码包和二进制包同时构建、只构建源码包、只构建二进制包 +- `-s`、`--show-dep-only`：仅展示target的依赖关系，不进行构建 +- `-k`、`--keep-all-rpms`：构建完成后保留所有构建出的rpm包，否则只保留target的rpm包 +- `-a`、`--auto-build`：自动构建target的所有递归依赖关系 +- `-b`、`--build-dep`：安装target的所有构建依赖 + +具体例子可参考[doc/examples.md](./doc/examples.md) diff --git a/tools/deb2rpm/config/deb2rpm_mapping.json b/tools/deb2rpm/config/deb2rpm_mapping.json new file mode 100644 index 0000000000000000000000000000000000000000..06169ea91aeb88c8f28057f2dab64ff16b7ba7a1 --- /dev/null +++ b/tools/deb2rpm/config/deb2rpm_mapping.json @@ -0,0 +1,76 @@ +{ + "debhelper": "debhelper", + "debhelper-compat": "debhelper", + "dpkg": "dpkg", + "dpkg-dev": "dpkg-devel", + "dpkg-perl": "dpkg-perl", + "po-debconf": "po-debconf", + "dh-autoreconf": "dh-autoreconf", + "pkg-config": "pkgconf", + "autoconf": "autoconf", + "automake": "automake", + "libc6": "glibc", + "libc6-dev": "glibc-devel", + "libbz2-1.0": "bzip2", + "libbz2-dev": "bzip2-devel", + "libexpat-dev": "expat-devel", + "libgd": "gd", + "libgd-dev": "gd-devel", + "libgeoip": "geoip", + "libgeoip-dev": "geoip-devel", + "libhiredis": "hiredis", + "libhiredis-dev": "hiredis-devel", + "libmaxminddb-dev": "libmaxminddb-devel", + "libmhash": "mhash", + "libmhash-dev": "mhash-devel", + "libpam0g": "pam", + "libpam0g-dev": "pam-devel", + "libpcre3": "pcre", + "libpcre3-dev": "pcre-devel", + "libperl": "perl", + "libperl-dev": "perl-devel", + "libssl": "openssl", + "libssl-dev": "openssl-devel", + "libxslt1": "libxslt", + "libxslt1-dev": "libxslt-devel", + "quilt": "quilt", + "zlib1g-dev": "zlib-devel", + "bison": "bison", + "libevent": "libevent", + "libevent-dev": "libevent-devel", + "libncurses5": "ncurses-libs", + "libncurses6": "ncurses-libs", + "libtinfo6": "ncurses-libs", + "libncursesw6": "ncurses-libs", + "ncurses-bin": "ncurses", + "libncurses-dev": "ncurses-devel", + "libncurses5-dev": "ncurses-devel", + "libncurses6-dev": "ncurses-devel", + "libutempter0": "libutempter", + "libutempter-dev": "libutempter-devel", + "libglib2.0": "glib2", + "libglib2.0-0": "glib2", + "libglib2.0-dev": "glib2-devel", + "texinfo": "texinfo", + "libtool": "libtool", + "libgpm": "gpm", + "libgpm-dev": "gpm-devel", + "libmd-dev": "libmd-devel", + "m4": "m4", + "perl": "perl", + "g++-multilib": "libstdc++-devel", + "gettext": "gettext", + "libacl1": "libacl", + "libacl1-dev": "libacl-devel", + "libselinux1": "libselinux", + "libselinux1-dev": "libselinux-devel", + "libattr1": "libattr", + "libattr1-dev": "libattr-devel", + "libjemalloc-dev": "jemalloc-devel", + "liblua5.1-dev": "lua-devel", + "liblzf-dev": "liblzf-devel", + "libsystemd-dev": "systemd-devel", + "procps": "procps-ng", + "tcl": "tcl", + "tcl-tls": "tcltls" +} diff --git a/tools/deb2rpm/config/resources.json b/tools/deb2rpm/config/resources.json new file mode 100644 index 0000000000000000000000000000000000000000..af06a42be9cb0f1d13ccc4102bba6373b9898e5c --- /dev/null +++ b/tools/deb2rpm/config/resources.json @@ -0,0 +1,79 @@ +{ + "refresh": false, + "repo_url": "https://repo.huaweicloud.com/ubuntu", + "main": { + "enable": true, + "Sources": { + "main": "/home/young/Source20/jammy/Sources-main", + "multiverse": "/home/young/Source20/jammy/Sources-multiverse", + "restricted": "/home/young/Source20/jammy/Sources-restricted", + "universe": "/home/young/Source20/jammy/Sources-universe" + }, + "Packages": { + "main": "/home/young/Package20/jammy/Packages-main", + "multiverse": "/home/young/Package20/jammy/Packages-multiverse", + "restricted": "/home/young/Package20/jammy/Packages-restricted", + "universe": "/home/young/Package20/jammy/Packages-universe" + } + }, + "updates": { + "enable": true, + "Sources": { + "main": "/home/young/Source20/jammy-updates/Sources-main", + "multiverse": "/home/young/Source20/jammy-updates/Sources-multiverse", + "restricted": "/home/young/Source20/jammy-updates/Sources-restricted", + "universe": "/home/young/Source20/jammy-updates/Sources-universe" + }, + "Packages": { + "main": "/home/young/Package20/jammy-updates/Packages-main", + "multiverse": "/home/young/Package20/jammy-updates/Packages-multiverse", + "restricted": "/home/young/Package20/jammy-updates/Packages-restricted", + "universe": "/home/young/Package20/jammy-updates/Packages-universe" + } + }, + "backports": { + "enable": true, + "Sources": { + "main": "/home/young/Source20/jammy-backports/Sources-main", + "multiverse": "/home/young/Source20/jammy-backports/Sources-multiverse", + "restricted": "/home/young/Source20/jammy-backports/Sources-restricted", + "universe": "/home/young/Source20/jammy-backports/Sources-universe" + }, + "Packages": { + "main": "/home/young/Package20/jammy-backports/Packages-main", + "multiverse": "/home/young/Package20/jammy-backports/Packages-multiverse", + "restricted": "/home/young/Package20/jammy-backports/Packages-restricted", + "universe": "/home/young/Package20/jammy-backports/Packages-universe" + } + }, + "security": { + "enable": true, + "Sources": { + "main": "/home/young/Source20/jammy-security/Sources-main", + "multiverse": "/home/young/Source20/jammy-security/Sources-multiverse", + "restricted": "/home/young/Source20/jammy-security/Sources-restricted", + "universe": "/home/young/Source20/jammy-security/Sources-universe" + }, + "Packages": { + "main": "/home/young/Package20/jammy-security/Packages-main", + "multiverse": "/home/young/Package20/jammy-security/Packages-multiverse", + "restricted": "/home/young/Package20/jammy-security/Packages-restricted", + "universe": "/home/young/Package20/jammy-security/Packages-universe" + } + }, + "proposed": { + "enable": true, + "Sources": { + "main": "/home/young/Source20/jammy-proposed/Sources-main", + "multiverse": "/home/young/Source20/jammy-proposed/Sources-multiverse", + "restricted": "/home/young/Source20/jammy-proposed/Sources-restricted", + "universe": "/home/young/Source20/jammy-proposed/Sources-universe" + }, + "Packages": { + "main": "/home/young/Package20/jammy-proposed/Packages-main", + "multiverse": "/home/young/Package20/jammy-proposed/Packages-multiverse", + "restricted": "/home/young/Package20/jammy-proposed/Packages-restricted", + "universe": "/home/young/Package20/jammy-proposed/Packages-universe" + } + } +} diff --git a/tools/deb2rpm/deb2rpm.py b/tools/deb2rpm/deb2rpm.py new file mode 100644 index 0000000000000000000000000000000000000000..f20d67c38949badc77c45ba96462cbec3b802523 --- /dev/null +++ b/tools/deb2rpm/deb2rpm.py @@ -0,0 +1,264 @@ +import argparse +import subprocess +import os +import re +import sys +import logging +import platform + +from typing import List + +from deb2rpm_manager import DRManager + +current_dir = os.path.dirname(os.path.abspath(__file__)) + +project_root = os.path.abspath(os.path.join(current_dir, "..")) +sys.path.append(project_root) + +repos = ['main', 'updates', 'backports', 'security', 'proposed'] + +logging.basicConfig(level=logging.INFO, + format='%(name)s - %(levelname)s - %(message)s') + +logger = logging.getLogger(__name__) + +home = os.getenv("HOME") +machine = platform.machine() + +type2param = { + "source_and_binary": "-ba", + "source": "-bs", + "binary": "-bb" +} + +def get_abs_path(path: str) -> str: + if os.path.isabs(path): + abs_path = path + else: + current_directory = os.getcwd() + abs_path = os.path.abspath(os.path.join(current_directory, path)) + + abs_path = os.path.expanduser(abs_path) + abs_path = os.path.normpath(abs_path) + + return abs_path + +def get_version(deb_version: str) -> str: + """ + 获取Epoch, Version, Release信息 + @param deb_version deb版本字符串 + """ + version_string = deb_version + epoch_pattern = r'^(\d+):' + release_pattern = r'(.*?)-(.*)' + + match_epoch = re.match(epoch_pattern, deb_version) + if match_epoch: + epoch = match_epoch.group(1) + version_string = version_string[len(epoch)+1:] + match_release = re.match(release_pattern, version_string) + if match_release: + release = match_release.group(2).strip() + version_string = version_string[:-len(release)-1] + return version_string + +def format_and_print_strings(strings, strings_per_row=3, string_width=20): + for i, string in enumerate(strings): + formatted_string = string.center(string_width) + + print(formatted_string, end=" ") + + if i < len(strings) - 1 and (i + 1) % strings_per_row != 0: + continue + else: + print() + +def check_packages_installed(packages: List) -> List: + uninstalled_list = [] + for package in packages: + try: + res = subprocess.run(['dnf', 'info', package], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode != 0: + uninstalled_list.append(package) + except Exception as e: + uninstalled_list.append(package) + + return uninstalled_list + +class DRClient: + def __init__(self, target: str, show_dep_only: bool, keep_all_rpms: bool, get_files_by_debs: bool, build_dep=False, auto_build=False, client_type="source_and_binary", install_list=None, spec_dep_json="", deep_dep_json=""): + self.target = target + self.keep_all_rpms = keep_all_rpms + self.manager = DRManager(target) + self.get_repo() + single_deps = self.manager.get_single_deps(self.chosen_repo, spec_dep_json) + if show_dep_only: + self.show_deps(single_deps) + sys.exit() + build_dep_list = [list(bd.split())[0] for bd in single_deps['Build-Depends'].keys() if single_deps['Build-Depends'][bd]] + if build_dep: + self.install_all_exists(build_dep_list) + sys.exit() + + not_installed = check_packages_installed(build_dep_list) + if len(not_installed) != 0: + logger.critical(f"以下依赖未安装: {not_installed}") + sys.exit() + if auto_build: + all_deps = self.manager.get_all_deps(self.chosen_repo, deep_dep_json) + self.install_all_exists(all_deps['exists']) + self.build_source(all_deps['source_order']) + self.genarate_spec(spec_dep_json, get_files_by_debs) + res = os.system(f"rpmbuild {type2param[client_type]} {self.spec_path}") + if res != 0: + print(f"构建{self.target}失败！") + sys.exit() + self.all_rpms = self.manager.get_all_rpms(self.chosen_repo, self.version) + logger.info(f"完成{self.target}的 {client_type} 构建!") + if client_type == "binary" and install_list != None and len(install_list) > 0: + logger.info("安装依赖: " + ' '.join(install_list)) + for i in install_list: + rpm_path = self.all_rpms[i] + command = f"sudo rpm -i {rpm_path}" + logger.info(f"安装{i}: 执行命令 {command}") + os.system(command) + + def get_repo(self): + all_repo_and_versions = self.manager.get_all_targets() + + if len(all_repo_and_versions) == 0: + print(f"未找到{self.target}!") + sys.exit() + + print(f"搜索到以下版本的{self.target}:") + cnt = 1 + for key, value in all_repo_and_versions.items(): + print(f"{cnt}: {key.center(10)} {value}") + cnt += 1 + + print(f"请选择所需要的版本，输入对应的标号(1~{cnt-1}):") + while True: + try: + num = int(input()) + if num >= 1 and num <= cnt - 1: + self.chosen_repo = list(all_repo_and_versions.keys())[num-1] + break + else: + print(f"输入越界！请重新输入对应标号(1~{cnt-1}):") + + except ValueError: + print(f"输入不合法！请重新输入对应标号(1~{cnt-1}):") + + self.version = get_version(all_repo_and_versions[self.chosen_repo]) + + def show_deps(self, single_deps): + print(f"{'#'*20}{(self.target + '构建依赖').center(20)}{'#'*27}") + print("Build-Depends:") + for i, whether_rpm in single_deps['Build-Depends'].items(): + if whether_rpm: + print(f" [RPM] {i}".center(60)) + else: + print(f" [DEB] {i}".center(60)) + print('#'*71) + for package in single_deps['Packages'].keys(): + print(f"{'#'*20}{(package + '运行依赖').center(20)}{'#'*27}") + for dep_type in single_deps['Packages'][package].keys(): + print(dep_type + ":") + for i, whether_rpm in single_deps['Packages'][package][dep_type].items(): + if whether_rpm: + print(f" [RPM] {i}".center(60)) + else: + print(f" [DEB] {i}".center(60)) + print('#'*71) + + + def install_all_exists(self, packages: List): + """ + 安装所有可直接安装或引入的依赖 + @param packages 所有可直接安装或引入的依赖列表 + """ + if len(packages) == 0: + print("没有可直接安装或引入的依赖！") + return + + command = 'sudo dnf install --nogpgcheck ' + ' '.join(packages) + logger.info("正在安装所有可直接引入或安装的依赖") + logger.info(f"执行命令: {command}") + os.system(command) + + def build_source(self, sources: List, force: bool): + """ + 编译所有需要额外编译的源码包 + @param sources 所有需要额外编译的源码包列表 + @param force 是否强制编译 + """ + if force: + return + for s in sources: + s_client = DRClient(s, show_dep_only=False, keep_all_rpms=self.keep_all_rpms, auto_build=True, client_type="binary", install_list=self.all_deps['source_and_package'][s]) + if not self.keep_all_rpms: + for rpm_path in s_client.all_rpms.values(): + os.system(f"rm -f {rpm_path}") + + def genarate_spec(self, json_path, get_files_by_debs): + print('#'*71) + logger.info(f"开始生成{self.target}的spec文件!") + self.manager.gen_spec(self.chosen_repo, get_files_by_debs) + self.spec_path = self.manager.add_dep2spec(json_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Build the target from deb to rpm") + # parser.add_argument("-n", "--newest", action="store_true") + group = parser.add_mutually_exclusive_group() + parser.add_argument("target", help="The target package") + parser.add_argument("-d", "--dep-json", help="Specify the path of json file with target dependency") + parser.add_argument("-D", "--deep-dep-json", help="Specify the path of json file with all recursive dependency of target") + parser.add_argument("-t", "--build-type", help="The type of building: source_and_binary(default) | source | binary") + group.add_argument("-s", "--show-dep-only", action="store_true", help="Only show the dependency infomation but not build") + parser.add_argument("-k", "--keep-all-rpms", action="store_true", help="Use if you want to keep all the rpms, else only keep rpms of target") + group.add_argument("-a", "--auto-build", action="store_true", help="Auto build the target recursively") + group.add_argument("-b", "--build-dep", action="store_true", help="Install all the exist build-depends") + + parser.add_argument("-g", "--get-files-by-debs", action="store_true", help="Fill the %%files in spec by downloading debs, deleted afterwards and generating quicker") + + args = parser.parse_args() + + show_dep_only = False + keep_all_rpms = False + auto_build = False + build_dep = False + get_files_by_debs = False + spec_dep_json = "" + deep_dep_json = "" + t = "source_and_binary" + + if args.show_dep_only: + show_dep_only = True + if args.keep_all_rpms: + keep_all_rpms = True + if args.auto_build: + auto_build = True + if args.build_dep: + build_dep = True + if args.get_files_by_debs: + get_files_by_debs = True + if args.dep_json: + spec_dep_json = get_abs_path(args.dep_json) + if args.deep_dep_json: + deep_dep_json = args.deep_dep_json + if args.build_type: + if args.build_type in type2param.keys(): + t = args.build_type + else: + print("Unsupported Type!") + sys.exit() + + client = DRClient(args.target, show_dep_only=show_dep_only, keep_all_rpms=keep_all_rpms, get_files_by_debs=get_files_by_debs, build_dep=build_dep, auto_build=auto_build, client_type=t, spec_dep_json=spec_dep_json, deep_dep_json=deep_dep_json) +<<<<<<< HEAD + +======= + # SpecGenarator(chosen_repo, args.target) + # dj = DepJson(chosen_repo, args.target) + +>>>>>>> b3ad248 (Add the option of downloading debs to get files) diff --git a/tools/deb2rpm/deb2rpm_manager.py b/tools/deb2rpm/deb2rpm_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..356a960ba5f9b38a59e370c9338ee14e7454303a --- /dev/null +++ b/tools/deb2rpm/deb2rpm_manager.py @@ -0,0 +1,150 @@ +import os +import sys +import json +import logging +import platform + +from typing import Dict, List + +current_dir = os.path.dirname(os.path.abspath(__file__)) + +project_root = os.path.abspath(os.path.join(current_dir, "..")) +sys.path.append(project_root) + +from scripts.init_db import InitDB +from scripts.spec import SpecGenarator +from scripts.get_dependency import DepJson +from scripts.recursive_depends import DeepDep +from scripts.json2spec import Json2Spec + +home = os.getenv("HOME") +machine = platform.machine() +mapping_path = f"{home}/deb2rpm/config/deb2rpm_mapping.json" +logger = logging.getLogger(__name__) + +repos = ['main', 'updates', 'backports', 'security', 'proposed'] + +class DRManager: + def __init__(self, target: str): + """ + 初始化Deb2Rpm Manager + @param target 目标包名 + @param save_dep_json 目标包依赖文件路径，默认为SPECS目录 + """ + self.target = target + with open(mapping_path, 'r') as file: + self.mapping = json.load(file) + + def get_all_targets(self) -> Dict: + """ + 从各个repo源中寻找目标包 + @return 返回各个repo源对应的目标包版本 + """ + db = InitDB() + all_repo_and_versions = {} + for repo in repos: + info = db.get_source_info(repo, self.target) + if info != None: + all_repo_and_versions[repo] = info[1] + + return all_repo_and_versions + + def get_single_deps(self, repo: str, json_path="") -> Dict: + dj = DepJson(repo, self.target, json_path=json_path) + data = {} + data["Build-Depends"] = {} + for bd in dj.dic["Build-Depends"]: + bd_name = list(bd.split())[0] + if bd_name in self.mapping.values(): + data["Build-Depends"][bd] = True + else: + data["Build-Depends"][bd] = False + + data["Packages"] = {} + for package in dj.dic["Packages"].keys(): + data["Packages"][package] = {} + for dep_type in dj.dic["Packages"][package].keys(): + data["Packages"][package][dep_type] = {} + for d in dj.dic["Packages"][package][dep_type]: + d_name = list(d.split())[0] + if d_name in self.mapping.values(): + data["Packages"][package][dep_type][d] = True + else: + data["Packages"][package][dep_type][d] = False + + if json_path != "": + dj.write_into_json() + + return data + + def get_all_deps(self, repo: str, json_path="") -> Dict: + """ + 获取repo中对应target的全量依赖 + @param repo 选择的repo源 + @param json_path 全量依赖json文件的保存路径，默认为不保存 + @return 返回所有依赖项 + [目标包依赖，可直接安装依赖，额外需要的依赖，额外需要编译的源码包] + """ + return_values = {} + # dj = DepJson(repo, self.target) + dd = DeepDep(repo, self.target, json_path=json_path) + # return_values["single_dep"] = dj.dic + return_values["exists"] = dd.exists + return_values["package_order"] = dd.package_order + return_values["source_order"] = dd.source_order + return_values["source_and_package"] = dd.source_and_package + + dd.write_into_json() + + return return_values + + def gen_spec(self, repo: str, download_debs: bool) -> str: + """ + 生成repo中对应target的spec文件 + @param repo 选择的repo源 + @return 返回spec文件保存路径 + """ + sg = SpecGenarator(repo, self.target, download_debs) + self.repo = repo + self.spec_file_path = sg.spec_file_path + return sg.spec_file_path + + def add_dep2spec(self, json_path="") -> str: + """ + 将依赖项填入生成的缺少依赖的spec文件 + 如果未指定依赖json文件保存路径，则在SPECS目录中生成后删除 + @return 返回完成的spec文件路径 + """ + remove_json = False + if json_path == "": + remove_json = True + specs_path = os.path.dirname(self.spec_file_path) + json_path = os.path.join(specs_path, f"{self.target}.json") + + dj = DepJson(self.repo, self.target, json_path=json_path) + dj.write_into_json() + + Json2Spec(self.spec_file_path, json_path) + logger.info(f"完成目标spec文件: {self.spec_file_path}") + + if remove_json: + os.remove(json_path) + else: + logger.info(f"生成依赖json文件: {json_path}") + + return self.spec_file_path + + def get_all_rpms(self, repo: str, version: str) -> Dict: + db = InitDB() + source_info = db.get_source_info(repo, self.target) + all_binary = source_info[4].strip().split() + binary2rpm = {} + + for binary in all_binary: + if binary == self.target: + rpm_name = f"{self.target}-{version}-1.{machine}.rpm" + else: + rpm_name = f"{self.target}-{binary}-{version}-1.{machine}.rpm" + binary2rpm[binary] = f"{home}/rpmbuild/RPMS/{machine}/{rpm_name}" + + return binary2rpm diff --git a/tools/deb2rpm/demos/deb2rpm/conf/config.yaml b/tools/deb2rpm/demos/deb2rpm/conf/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/conf/logger.conf b/tools/deb2rpm/demos/deb2rpm/conf/logger.conf new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/conf/pkg_name_maaping.json b/tools/deb2rpm/demos/deb2rpm/conf/pkg_name_maaping.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/deb2rpm.py b/tools/deb2rpm/demos/deb2rpm/deb2rpm.py new file mode 100644 index 0000000000000000000000000000000000000000..57ec1dbbdde2bca43fc86dec8bdbdcb695485592 --- /dev/null +++ b/tools/deb2rpm/demos/deb2rpm/deb2rpm.py @@ -0,0 +1,58 @@ +import logging +import argparse +import os.path +import queue + +from src.data import Data +from src.package import Deb, Depend +from util.logger import Logger +from util.yaml_handler import YamlHandler +from util.deb_helper import Debhelper + +Logger.init_logger() +logger = logging.getLogger('deb2rpm') + + +def do_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("-c", "--config_path", default="./conf/config.yaml", help="config yaml path") + parser.add_argument("pkg", type=str, help="The Debina Package Name") + + return parser + + +def normalize_deb_depend(deb: Deb, data: Data): + for depend in deb.depend_list: + version = depend.version + if depend.pkg.endswith(':any'): + depend.set_pkg(depend.pkg[:-4]) + if not version: + continue + if ':' in version: + version = version.split(':')[-1] + num_depend = version.count('-') + version_deb = data.get_deb_bersion_by_package(deb.Package) + num_deb = version_deb.count('-') + if num_deb and num_deb == num_depend: + version = version.split('-')[0] + if depend.flag in ('<', '<<'): + depend.set_flag('<=') + depend.set_depend_version(version) + + +def pkg_name_mapping(pkg: str, depend_pkg: str, depend: Depend, data: Data): + pass + + +def breadth_first_search_package(pkg_list: list, data: Data): + pass + + +def main(): + parser = do_args() + pass + + +if __name__ == "__main__": + main() diff --git a/tools/deb2rpm/demos/deb2rpm/src/data.py b/tools/deb2rpm/demos/deb2rpm/src/data.py new file mode 100644 index 0000000000000000000000000000000000000000..10445dc63a99549baf8c6f807cae5a8c2ecd6274 --- /dev/null +++ b/tools/deb2rpm/demos/deb2rpm/src/data.py @@ -0,0 +1,32 @@ +class Data: + + def __init__(self, config: dict): + self.config = config + self.deb_db = None + self.src_deb_db = None + self.rpm_db = None + self.pkg_name_mapping_db = None + self.load_database() + + def load_database(self): + db_path = os.path.join(self.config['workdir'], 'database') + deb_primary, rpm_primary = self.download_database(db_path) + + def download_database(self, db_path): + pass + + def normalize_deb_json_database(self, file_list: list): + pass + + def normalize_source_deb_database(self, dic: dict): + pass + + def normalize_deb_provides_database(self, dic: dict): + pass + + def normalize_rpm_sqlite_database(self, file_list: list): + pass + + def normalize_pkg_name_mapping_database(self, file_list: list): + pass + diff --git a/tools/deb2rpm/demos/deb2rpm/src/package.py b/tools/deb2rpm/demos/deb2rpm/src/package.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/util/command.py b/tools/deb2rpm/demos/deb2rpm/util/command.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/util/deb_helper.py b/tools/deb2rpm/demos/deb2rpm/util/deb_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/util/logger.py b/tools/deb2rpm/demos/deb2rpm/util/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/demos/deb2rpm/util/yaml_handler.py b/tools/deb2rpm/demos/deb2rpm/util/yaml_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tools/deb2rpm/doc/classes.drawio.png b/tools/deb2rpm/doc/classes.drawio.png new file mode 100755 index 0000000000000000000000000000000000000000..8d151b94850be9b45282cde1bf24637e20c996f9 Binary files /dev/null and b/tools/deb2rpm/doc/classes.drawio.png differ diff --git a/tools/deb2rpm/doc/db_process.drawio.png b/tools/deb2rpm/doc/db_process.drawio.png new file mode 100644 index 0000000000000000000000000000000000000000..ac05480257acd0aa50445e5653f7d14b72821dec Binary files /dev/null and b/tools/deb2rpm/doc/db_process.drawio.png differ diff --git a/tools/deb2rpm/doc/deb2rpm_doc.md b/tools/deb2rpm/doc/deb2rpm_doc.md new file mode 100644 index 0000000000000000000000000000000000000000..eb59b321a29c177f5b7eef4e4d4d3f1a816c6ec1 --- /dev/null +++ b/tools/deb2rpm/doc/deb2rpm_doc.md @@ -0,0 +1,268 @@ +# deb2rpm +[TOC] +## 1. 需求 +1. 功能需求：将deb软件包转换为能够在openEuler 22.03-LTS上运行的rpm软件包以及src.rpm源码包 +2. 兼容需求：转换出的rpm软件包和包含spec文件的src.rpm源码包，要能在openEuler及openEuler的衍生发行版上正常构建和运行 +3. 自动化需求：转换工具能够通用性自动化地转换大部分deb软件包 + +## 2. 需要解决的困难 +1. deb软件包和rpm软件包的文件结构和部分语法不同，需要进行内容的映射，部分独有的内容需要特殊处理 +2. deb软件包和rpm软件包使用两套包管理器，其依赖对象不同，需要进行依赖的映射 + +## 3. 方案总体设计 +### 3.1 spec模板 +rpm包的关键内容就是spec文件，spec文件的基本格式如下： +```spec +################################################################## +# 1. 元数据信息 +################################################################## +Name: +Version: +Release: +Summary: +License: +URL: +Source0: +... +Patch0: +... + +%description + +################################################################## +# 2. 依赖信息 +################################################################## +BuildRequires: +Requires: +... + +################################################################## +# 3. 安装信息 +################################################################## +%prep + +%build + +%install + +################################################################## +# 4. 子包信息 +################################################################## +%package -n +Summary: +Requires: + +%description -n + +################################################################## +# 5. 文件信息 +################################################################## +%files +... + +%files -n +... + +################################################################## +# 6. changelog +################################################################## +%changelog +... +``` + +### 3.2 类图 +要从不同文件的不同位置提取出以上构建spec文件的内容，本工具中包含的类的总体设计如下图所示： +![classes](./classes.drawio.png) +上图中各个类的基本功能如下： +- `DscParser`：解析dsc文件，获取主软件包的元数据信息 +- `ControlParser`：解析control文件，获取子包的元数据信息 +- `LicenseParser`：解析license文件，获取license信息 +- `DebFiles`：获取文件信息 +- `Spec`：将获取到的deb信息转化为spec文件的格式 +- `BaseDB`: 调用sqlite数据库的接口 +- `SourceDB`：从Source文件(如[https://repo.huaweicloud.com/ubuntu/dists/jammy/main/source/Sources.gz](https://repo.huaweicloud.com/ubuntu/dists/jammy/main/source/Sources.gz))构建数据库 +- `PackageDB`：从Package文件(如[https://repo.huaweicloud.com/ubuntu/dists/jammy/main/binary-amd64/Packages.gz](https://repo.huaweicloud.com/ubuntu/dists/jammy/main/binary-amd64/Packages.gz))构建数据库 +- `InitDB`: 获取源文件及依赖的数据库 + +类与spec文件内容的对应关系如下： +| 类 | spec | +| :---: | :---: | +| InitDB | 依赖相关 | +| DscParser | Name, Epoch, Version, Release, URL, Source | +| ControlParser | Summary, %description, 子包名 | +| LicenseParser | License | +| DebParser | Patch | +| DebFiles | %build, %files | +| SpecGenarator | %prep, %install, %changelog | + +### 3.3 整体流程 +1. 初始化或更新数据库信息 +2. 在`SOURCES`目录下创建与软件包同名的文件夹，并根据数据库中的源文件信息获取deb源码包，并解压得到源码文件夹 +3. `DscParser`解析dsc文件获取主软件包的元数据信息 +4. `ControlParser`解析control文件获取子包的元数据信息 +5. `LicenseParser`解析license文件获取license信息 +6. 从debian/patches文件夹中获取补丁文件并复制到步骤二创建的文件夹中 +7. 调用`get_files`脚本获取安装文件列表 +8. 将元上述信息转化为spec中对应的内容 +9. 生成spec文件 +10. 将Source和Patch文件复制到`SOURCES`中，并删除步骤二创建的文件夹 + +## 4. 具体实现 +下面按照spec文件的各个模块来介绍具体的实现方法。 +### 4.1 Source和依赖相关内容 +本部分是从Source文件和Package文件中的已有内容获取的，用户通过改写`resources.json`文件中的文件路径来设置使用的本地文件，该json文件模板如下： +```json +{ + "refresh": false, + "Sources": { + "main": "/path/to/main/source", + "multiverse": "/path/to/multiverse/source", + "restricted": "/path/to/restricted/source", + "universe": "/path/to/universe/source" + }, + "Packages": { + "main": "/path/to/main/package", + "multiverse": "/path/to/multiverse/package", + "restricted": "/path/to/restricted/package", + "universe": "/path/to/universe/package" + } +} +``` +为了减少重复的读写并方便做设置修改，数据库中除了将以上八个文件分别对应到八张表，还另设一张表用于存储文件路径信息。当json文件中某个文件的路径信息被修改了后，再次运行程序前会将对应的表更新为新文件，数据库初始化部分的流程图如下： +![db_process](./db_process.drawio.png) +数据库存储内容如下： +- path表： source和package文件路径信息 +- SourceMain, SourceMultiverse, SourceRestricted, SourceUniverse: +存储Source文件中后续需要的信息，包含源码包所在源的文件夹、构建依赖(*Build-Depends*)、所需要的源文件 +- PackageMain, PackageMultiverse, PackageRestricted, PackageUniverse: +存储Package文件中后续需要的信息，即各个软件包(包括子包)的依赖信息，包括`Provide`, `Pre-Depends`, `Depends`, `Recommends`, `Suggests`, `Breaks`, `Replaces` + +完成了数据库内容的获取，就可以将获取信息的接口`get_source_info`和`get_package_info`封装到`InitDB`类中了。 + +### 4.2 构建build目录 +获取到source信息后，工具就会在`SOURCES`目录下的临时文件夹中下载这些文件： +```python +source_name, build_requires, dir_path, source_files = get_source_info(package_name) + +web_repo = "https://repo.huaweicloud.com/ubuntu" +for file in source_files.split(): + os.system(f"wget {web_repo}/{dir_name}/{file}") +``` +随后执行命令`dpkg-source -x --no-check `来构建源码文件夹(build目录)。相比于`tar`的相关解压方式，该方案的优势在于： +- 无需考虑源码压缩包的多种压缩文件格式 +- 无需考虑解压路径的设置 +- 可进行任意数量的文件解压 +- 作为`apt source`的实际命令之一，`dpkg-source -x`会自动执行`debian/patches/series`中列出的补丁文件 + +### 4.3 Name, Version, URL, Sources +本部分内容由`DscParser`提取，由于dsc文件格式较为统一，因此这里使用了一个简单的信息提取函数： +```python +def get_line_info(line: str) -> List: + pattern = r'^(.*?):\s*(.*)' + match = re.search(pattern, line) + if match: + return match.group(1).strip(), match.group(2).strip() + return "", "" +``` +即如果一行字符串为 +``` +Version: 1.0.0 +``` +该函数就会返回`["Version", "1.0.0"]`，从而提取出需要的信息。 + +后续在`Spec`类中会对`Version`信息做进一步处理，因为deb中`Version`的格式为`[Epoch:]Version[-Release]`，分别对应spec中的三个字段，其中`Epoch`和`Release`是可选的，在本工具中`Epoch`默认为无，`Release`默认为`1` + +这里不从source文件而从dsc文件中获取这些信息的原因在于减少对于数据库的依赖，并且为将一个未收集信息的deb包转换为rpm包提供可能。 + +### 4.4 %package, BuildArch, Summary, %description +本部分内容由`ControlParser`获取，control文件格式类似于dsc文件，因此也可以采用`get_line_info`的方法。但`Summary`和`%description`信息有所不同，在control文件中，前者对应`Description`字段的第一行内容，后者对应该字段的剩余内容，而该字段后的内容要么就是下一个Package，要么就是文件结尾，因此可以如下处理： +```python +with open(path, 'r') as file: + package_name = "" + description = "" + + for line in file: + key, value = get_line_info(line) + if key == "Package": + description = False + package_name = value + self.package_data[package_name] = {} + if key == "Description": + self.package_data[package_name].update({"Summary": value}) + description = True + self.package_data[package_name]["description"] = "" + elif description: + self.package_data[package_name]["description"] += line +``` + +### 4.5 License +本部分由`LicenseParser`提取，根据deb包license文件的格式[规范](https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/)，该文件总体的格式大致为： +``` +Files: * +Copyright: 1975-2010 Ulla Upstream +License: GPL-2+ + +Files: debian/* +Copyright: 2010 Daniela Debianizer +License: GPL-2+ + +Files: debian/patches/fancy-feature +Copyright: 2010 Daniela Debianizer +License: GPL-3+ + +Files: */*.1 +Copyright: 2010 Manuela Manpager +License: GPL-2+`` +``` +此处仅提取了`Files: *`下的`License`字段值填写spec文件中对应的`license`名。 +但也存在不满足这种规范的软件包，如`zip`。由于spec文件中`license`字段不可缺少，暂时将这种软件包的`license`填写为MIT + +### 4.6 Patch +本部分内容由`DebParser`获取。如前所述，补丁文件在构建build目录时会自动执行，因此在spec文件中并不需要手动执行补丁，但仍然需要列出补丁文件。事实上，deb包的补丁文件均在`debian/patches`文件夹下，因此只需获取该目录下所有的`.patch`和`.diff`文件即可。 + +### 4.7 %build, %files +本部分内容由`DebFiles`获取，由于deb包并不像rpm包这样显式地列出待安装目录，而是将各子包的待安装目录先安装在debian目录下。因此本工具会在`SOURCES`目录下创建的临时文件夹中完成deb包所有文件结构的构建，进而获取到各子包需要安装的文件。 + +尝试发现，在执行`fakeroot debian/rules binary`时，不同的包会执行不同的命令，但如果在中间插入无关的指令实际上并不会产生别的影响。因此，可以先确定一个通用的有序的全体命令集。 + +另一方面，`debian/rules`文件中可能含有部分命令的重载，如`override_dh_auto_install`，这时就需要将全体命令集中的`fakeroot dh_auto_install`替换为`fakeroot debian/rules dh_auto_isntall`。 +有时也存在两个替换的情况，即`fakeroot dh_install`替换为`fakeroot debian/rules dh_install-arch`和`fakeroot debian/rules dh_install-indep`。 + +通过对`debian/rules`文件的解析，就能获得文件信息的全部命令，也就是`%build`对应的内容。 + +### 4.8 %prep, %install +这两部分内容相对固定，由`SpecGenarator`类直接写入。 +`%prep`部分需要实现源码压缩包的解压，这里除了采用`dpkg-source`的解压方法，为了使得生成的spec文件更有通用性，本工具在此处采用tar命令的解压方法。需要在脚本中根据源码压缩包类型确定命令，具体如下： +| 压缩包类型 | 命令 | +| :---: | :---: | +| .tar.gz | tar -xzf | +| .tar.bz2 | tar -xjf | +| .tar.xz | tar -xJf | +```spec +%prep +rm -rf %{_builddir}/%{name}-%{version} +mkdir %{_builddir}/%{name}-%{version} +tar -xzf %{SOURCE0} -C %{_builddir}/%{name}-%{version} --strip-components=1 # 避免了解压文件夹名的不确定性 +tar -xJf %{SOURCE1} -C %{_builddir}/%{name}-%{version} + +cd %{_builddir}/%{name}-%{version} +%patch0 -p1 +... +``` +从而实现了rpm默认的build行为，即在`%{_builddir}`下创建名为`%{name}-%{version}`的目录，并在其中完成build +由于本阶段没有使用`setup`等默认的宏，因此`%build`的默认路径仍然是`%{_builddir}`, 因此需要在`%build`阶段最开始加上`cd %{name}-%{version}` + +`%install`部分需要实现的是将待安装文件全部复制到`%{buildroot}`目录下，由于需要实现的是文件树结构的复制，本工具采用的是`rsync`命令： +```spec +rm -rf %{buildroot}/* +rsync -a --ignore-existing --exclude='DEBIAN/' %{_builddir}/%{name}-%{version}/debian//* %{buildroot}/%{name}-%{version}-{release}. +... +``` + +### 4.9 %changelog +本部分内容暂时为统一格式，模板如下： +```spec +%changelog +* <星期> <月份> <日> <年份> OpenEuler %{version}-%{release} +- Build %{name} from deb to rpm +``` \ No newline at end of file diff --git a/tools/deb2rpm/doc/design.md b/tools/deb2rpm/doc/design.md new file mode 100755 index 0000000000000000000000000000000000000000..6ebb3f67b85496769f8daf99a2bb684645729dbc --- /dev/null +++ b/tools/deb2rpm/doc/design.md @@ -0,0 +1,26 @@ +# deb2rpm +本部分完成了在Ubuntu环境下deb源码包生成对应spec文件，从而构建rpm包及rpm源码包。其中类的设计如下图所示： +![classes](./classes.drawio.png) + +## 整体流程 +0. 在`~/rpmbuild/SOURCES`文件夹下创建源代码文件夹，并在该文件夹下调用`apt source` +1. `DscParser`解析dsc文件获取主软件包的元数据信息 +2. `ControlParser`解析control文件获取子包的元数据信息 +3. `LicenseParser`解析license文件获取license信息 +4. 从debian/patches文件夹中获取补丁文件并复制到源代码文件夹中 +5. 调用`get_files`脚本获取安装文件列表，并完成build +6. 将元数据信息转化为spec中对应的内容 +7. 生成spec文件 +8. 将待安装文件目录复制到BUILDROOT中(即`%install`) +9. 将Source和Patch文件复制到SOURCES中 + +## 测试结果 +在Ubuntu 20.04环境下，对`apache2`、`nginx`、`redis`、`ffmpeg`、`tmux`、`nghttp2`、`zip`、`dh-python`进行了测试，均能够成功构建出rpm包及rpm源码包。 + +## 部分待处理细节 +1. **License的提取:** 目前忽略了子软件包可能存在的不同License，并且对于特殊的License文件格式未处理，默认为MIT，如zip +2. **BuildArch:** 忽略了`BuildArch`的内容，因为使用`noarch`在建包时可能会出现有架构要求的二进制文件而出错的情况，如nghttp2 + +## 迁移到openEuler需要解决的问题 +1. **源码包**：源码包对应文件可以在Source池中查询，由于该文件较大，可以考虑事先存储在数据库中 +2. **依赖**：需要填写依赖相关字段 \ No newline at end of file diff --git a/tools/deb2rpm/doc/examples.md b/tools/deb2rpm/doc/examples.md new file mode 100644 index 0000000000000000000000000000000000000000..e7998bfcd1ac0db2336de49fbb95236f7cff93f6 --- /dev/null +++ b/tools/deb2rpm/doc/examples.md @@ -0,0 +1,137 @@ +# Examples of deb2rpm + +此处以`zip`和`screen`两个包为例，展示`deb2rpm`的使用方法。 + +## zip +### `python deb2rpm.py -s zip` 查看依赖 +``` +搜索到以下版本的zip: +1: main 3.0-12build2 +请选择所需要的版本，输入对应的标号(1~1): +``` +只有一个版本的`zip`，输入`1`继续。 + +``` +#################### zip构建依赖 ########################### +Build-Depends: + [RPM] bzip2-devel +####################################################################### +#################### zip运行依赖 ########################### +Depends: + [RPM] bzip2 + [RPM] glibc >= 2.34 +Recommends: + [DEB] unzip +####################################################################### + +``` +上述输出提供了`zip`源码包的构建依赖，以及`zip`二进制包的运行依赖和推荐依赖。其中`[RPM]`表示可直接安装或引入的依赖，`[DEB]`表示需要额外安装的依赖。 + +### `python deb2rpm.py -b zip`安装`zip`包构建依赖 +类似于`apt-get build-dep`安装所有构建依赖，执行 +``` +sudo dnf install --nogpgcheck bzip2-devel +``` +加上`--nogpgcheck`是因为有一些依赖并不在`openEuler`的官方源中，而是通过`oepkgs`的源提供的。 + +### `python deb2rpm.py zip`构建`zip`包 +构建前程序会自动检查构建依赖是否已全部安装，如果没有安装会提示安装，如果已安装则会自动下载`zip`的源码包并进行构建，默认执行source_and_binary构建，构建成功后会在`~/rpmbuild`目录下按照RPM包的目录结构生成`zip`的RPM包。 + +## screen +### `python deb2rpm.py -s screen` 查看依赖 +``` +搜索到以下版本的screen: +1: main 4.9.0-1 +请选择所需要的版本，输入对应的标号(1~1): +1 +#################### screen构建依赖 ############################ +Build-Depends: + [RPM] dpkg-devel >= 1.16.1 + [RPM] ncurses-devel + [RPM] pam-devel + [RPM] libutempter-devel + [RPM] texinfo +####################################################################### +#################### screen运行依赖 ########################### +Depends: + [RPM] glibc >= 2.34 + [DEB] libcrypt1 >= 4.1.0 + [RPM] pam >= 0.99.7.1 + [RPM] ncurses-libs >= 6 + [RPM] libutempter >= 1.1.5 +Suggests: + [DEB] byobu + [DEB] ncurses-term +#################### screen-udeb运行依赖 ########################### +####################################################################### +``` + +`screen`的依赖更为复杂，且具有一定的代表性。除了`zip`中的三类依赖外，`screen`需要额外安装的依赖还有`libcrypt1`，因此需要额外编译`libcrypt1`的源码包`libxcrypt`来安装`libcrypt1`。 + +``` +python deb2rpm.py screen -d ~/screen_dep.json +``` +```json +{ + "Name": "screen", + "Build-Depends": [ + "dpkg-devel >= 1.16.1", + "ncurses-devel", + "pam-devel", + "libutempter-devel", + "texinfo" + ], + "Packages": { + "screen": { + "Depends": [ + "glibc >= 2.34", + "libcrypt1 >= 4.1.0", + "pam >= 0.99.7.1", + "ncurses-libs >= 6", + "libutempter >= 1.1.5" + ], + "Suggests": [ + "byobu", + "ncurses-term" + ] + }, + "screen-udeb": {} + } +} +``` + +如果需要保存`screen`的全量依赖信息(即所有递归依赖)，可以指定`-a`或`--all-dep-json`参数，构建完成后对应json依赖文件会保存在指定目录下，例如： +``` +python deb2rpm.py screen -a ~/screen_all_dep.json +``` +```json +{ + "screen": { + "Depth": 0, + "Build-Depends": [ + "debhelper-compat = 13", + "dpkg-dev >= 1.16.1", + "libncurses-dev", + "libpam0g-dev", + "libutempter-dev", + "texinfo" + ], + "Pre-Depends": [], + "Depends": [ + "libc6 >= 2.34", + "libcrypt1 >= 4.1.0", + "libpam0g >= 0.99.7.1", + "libtinfo6 >= 6", + "libutempter0 >= 1.1.5" + ] + }, + "libcrypt1": { + "Depth": 1, + "Build-Depends": [], + "Pre-Depends": [], + "Depends": [ + "libc6 >= 2.25" + ] + } +} +``` diff --git a/tools/deb2rpm/docs/assets/servers.png b/tools/deb2rpm/docs/assets/servers.png new file mode 100644 index 0000000000000000000000000000000000000000..b211cea3bebb154e9452dccf06dc0a6fcac87fd2 Binary files /dev/null and b/tools/deb2rpm/docs/assets/servers.png differ diff --git a/tools/deb2rpm/docs/assets/user-case.png b/tools/deb2rpm/docs/assets/user-case.png new file mode 100644 index 0000000000000000000000000000000000000000..cbb12ebb936da2cb7ef4fea9ce90f4a632a5d9f0 Binary files /dev/null and b/tools/deb2rpm/docs/assets/user-case.png differ diff --git a/tools/deb2rpm/docs/design_docs/deb2rpm-design.md b/tools/deb2rpm/docs/design_docs/deb2rpm-design.md new file mode 100644 index 0000000000000000000000000000000000000000..d9dc65a1cf9ca91509435e9a0a121e8ae74054c9 --- /dev/null +++ b/tools/deb2rpm/docs/design_docs/deb2rpm-design.md @@ -0,0 +1,79 @@ +[TOC] + +## 1. 需求描述 + +1. 迁移需求： 将 Ubuntu（18.04）的 deb 软件包，通过工具自动化转化为在openEuler 20.03-LTS-SP3 上可运行的 rpm 软件包 +2. 兼容性需求： 转化出的 rpm 软件包，要求通过构建测试、兼容性测试 +3. 升级需求： 支持自动化迭代提升构建成功率 + +### 1.1 依赖组件 + +| 组件 | 组件描述 | 可获得性 | +| --------- | ------ | --------- | +| python3 | python3 及以上版本 | 可使用 `dnf/yum` 进行安装 | +| python3-concurrent-log-handler | logging日志辅助模块，用于日志转储 | 可使用 `dnf/yum` 进行安装 | +| dpkg | 系统上处理 deb 软件包的基础指令 | 可使用 `dnf/yum` 进行安装 | +| dpkg-devel | 提供一些开发工具，包括 dpkg-source | 可使用 `dnf/yum` 进行安装 | +| debhelper | 提供一些构建 deb 包的工具 | 配置 oepkgs 镜像源，可使用 `dnf/yum` 进行安装 | +| ruby | ruby 2.7 及以上版本 | 可使用 `dnf/yum` 进行安装 | +| ruby-devel | 提供一些 ruby 开发工具 | 可使用 `dnf/yum` 进行安装 | +| rubygems | ruby 的包管理器，功能上类似于 apt-get、yum 等 | 可使用 `dnf/yum` 进行安装 | +| fpm | linux 下的一款开源打包工具，可实现 deb 包转化成 rpm 包 | 可使用 `gem` 进行安装 | +| rpmrebuild | 用于构建 rpm 包的工具 | 可使用 `dnf/yum` 进行安装 | + +### 1.3 License + +Mulan V2 + +## 2. 设计概述 + +### 2.1 整体方案分析 + +![](../assets/servers.png) + +### 2.2 设计原则 + +- 数据与代码分离：将需求动态改变的数据提取到配置文件，而非用硬编码写死在代码里 +- 接口与实现分离：外部依赖模块接口而不是依赖模块实现 +- 模块划分：模块之间互相独立，执行互相不影响 + +## 3. 需求分析 + +### 3.1 分析思路 + +- 特性分析 + 用户拿到deb2rpm工具，有不同的功能诉求，不同的功能诉求确定了deb2rpm实现逻辑上的差异： + 1. 构建二进制rpm包：输入是一个deb包，想基于deb2rpm-server的OS环境构建出一个rpm包，同时提供 “忽略安装依赖” 的构建参数，以保证构建生成的rpm，在安装时不会由于缺少安装依赖无法安装 + 2. 构建二进制rpm包及src.rpm包：输入是一个deb包，想基于deb2rpm-server的OS环境，通过工具转换生成spec文件，从而构建生成rpm包以及src.rpm包，转换生成的spec文件，通过包名映射数据表，修改软件包的构建及安装依赖包名，以提高构建成功率 + 3. deb包转化依赖扫描：输入是一个deb包，基于deb2rpm-client以及deb2rpm-server环境进行构建、安装依赖扫描，识别依赖是否存在缺失 + 4. 构建出的rpm包镜像源发布：将构建好的rpm存放在固定目录，并构建自己的镜像源，用于后续依赖问题解决 + +- 用例视图 + +![](../assets/user-case.png) + + +#### 3.1.1 deb2rpm-client + +- 模块介绍 + 获取deb包的源码，通过处理源码包中的控制文件，解析deb软件包的构建依赖，基于包名映射数据表解决构建依赖。通过debhelper工具，完成软件包编译 + +#### 3.1.2 deb2rpm-server + + +### 3.2 框架设计 +``` +. +├── deb2rpm +├── scripts 工具脚本 +├── server 服务端 +├── config 配置文件 +│ ├── version.config deb2rpm 版本信息配置文件, 用于后续版本演进 +│ ├── source.list deb 软件包镜像源获取地址 +│ ├── deb2rpm.repo rpm 软件包镜像源获取地址 +│ └── debian2openeuler.json 包名映射配置文件 +├── templates 提供配置文件，被测试框架 lkp-tests 所集成 +└── tests 测试用例 +``` + + diff --git a/tools/deb2rpm/initialize.sh b/tools/deb2rpm/initialize.sh new file mode 100755 index 0000000000000000000000000000000000000000..9297f8808fa90fa3294f581aa8e435d1c39bae0a --- /dev/null +++ b/tools/deb2rpm/initialize.sh @@ -0,0 +1,69 @@ +#!/bin/bash +deb2rpm_directory="$(dirname "$0")" +home_dir="$HOME" + +version="" +url="" + +while getopts ":v:u:" opt; do + case $opt in + v) + version="$OPTARG" + ;; + u) + url="$OPTARG" + ;; + \?) + echo "Illegal param: -$OPTARG" >&2 + exit 1 + ;; + esac +done + +shift $((OPTIND-1)) + +if [ -n "$version" ]; then + echo "Specify version: $version" +else + version="jammy" + echo "Default version: $version" +fi + +if [ -n "$url" ]; then + echo "Specify url: $url" +else + url="https://repo.huaweicloud.com/ubuntu" + echo "Default url: $url" +fi + +echo "Ensure the project is under your home directory." +mv -n "$deb2rpm_directory" "$home_dir" + +echo "Install the required dependency for deb2rpm" +echo "Installing python and networkx" +sudo dnf install python3 python3-pip +pip install networkx + +echo "Installing rpm-build" +sudo dnf install rpm-build + +echo "Add repos of oepkgs to enable deb" +sudo dnf config-manager --add-repo https://repo.oepkgs.net/openeuler/rpm/openEuler-22.03-LTS/extras/aarch64/ +sudo dnf config-manager --add-repo https://repo.oepkgs.net/openeuler/rpm/openEuler-22.03-LTS/compatible/aur/x86_64/ +sudo dnf config-manager --add-repo https://repo.oepkgs.net/openeuler/rpm/openEuler-22.03-LTS/compatible/f33/x86_64/ +dnf clean all && dnf makecache +sudo dnf install --nogpgcheck debhelper dpkg dpkg-devel + +echo "Get repo files of $url" +python3 ~/deb2rpm/scripts/get_repos.py "$version" "$url" + +echo "Initialize the database" +if [ ! -f "$home_dir/deb2rpm/database/deb2rpm.db" ]; then + touch ~/deb2rpm/database/deb2rpm.db + echo "Create a database: ~/deb2rpm/database/deb2rpm.db" +else + echo "Database exists" +fi + +python3 ~/deb2rpm/scripts/init_db.py deb2rpm_test +echo "Finishing initializing!" diff --git a/tools/deb2rpm/scripts/deb_parser.py b/tools/deb2rpm/scripts/deb_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..c19fea3485db1fc061e494959426e8fa32dbd731 --- /dev/null +++ b/tools/deb2rpm/scripts/deb_parser.py @@ -0,0 +1,166 @@ +import os +import re +import glob +import shutil +import logging + +from typing import Union, List + +logger = logging.getLogger(__name__) +dsc_keywords = ['Source', 'Version', 'Homepage'] + +def get_line_info(line: str) -> List: + """ + 从line中提取信息 + @param line 一行，格式为key: value + @return 返回[key, value] + """ + pattern = r'^(.*?):\s*(.*)' + match = re.search(pattern, line) + if match: + return match.group(1).strip(), match.group(2).strip() + return "", "" + +class DscParser: + def __init__(self, path: str): + """ + 初始化DscParser，获取元数据和Source信息 + @param path .dsc文件路径 + """ + self.meta_data = {} + + get_file = False + with open(path, 'r') as file: + for line in file: + key, value = get_line_info(line) + if key == 'Checksums-Sha256': + get_file = False + break + if line.strip() == "": + get_file = False + elif key in dsc_keywords: + self.meta_data[key] = value + elif key == 'Checksums-Sha1': + get_file = True + self.meta_data['Files'] = [] + elif get_file: + parts = line.strip().split() + self.meta_data['Files'].append(parts[-1]) + +class ControlParser: + def __init__(self, path: str): + """ + 初始化ControlParser，获取子包信息 + @param path control文件路径 + """ + self.package_data = {} + + with open(path, 'r') as file: + package_name = "" + description = False + + for line in file: + key, value = get_line_info(line) + if key == "Package": + description = False + package_name = value + self.package_data[package_name] = {} + elif key == "Architecture": + if value == 'any' or value == 'all': + self.package_data[package_name].update({key: "noarch"}) + else: + self.package_data[package_name].update({key: value}) + elif key == "Description": + self.package_data[package_name].update({"Summary": value}) + description = True + self.package_data[package_name]["description"] = "" + elif description: + self.package_data[package_name]["description"] += line + elif key != '' and value != '': + logger.warning(f'Ignore {key}: {value}') + +class LicenseParser: + def __init__(self, path: str): + """ + 初始化LicenseParser，获取License名(默认MIT) + @param path copyright文件路径 + """ + self.license = "" + + if os.path.exists(path): + capture = False + with open(path, 'r') as file: + for line in file: + if line.strip() == "Files: *": + capture = True + continue + if capture: + match = re.search(r'License: (.*)', line) + if match: + self.license = match.group(1).strip() + break + + if not capture: + self.license = 'MIT' + else: + self.license = 'MIT' + + +class DebParser: + def __init__(self, path: str, build_path: str): + """ + 初始化DebParser + @param path 源码包目录(即包含dsc的目录) + @param build_path build目录 + """ + self.meta_data = {} + self.control_data = {} + + dsc_files = self.get_certain_files(path, ".dsc", False) + + assert len(dsc_files) == 1, 'dsc错误！' + self.dsc_name = dsc_files[0] + dsc = DscParser(self.dsc_name) + + control = ControlParser(os.path.join(build_path, "debian/control")) + lic = LicenseParser(os.path.join(build_path, "debian/copyright")) + + # 获取patch并复制到源文件夹中 + patch_path = os.path.join(build_path, "debian/patches") + self.patches = [] + if os.path.exists(patch_path): + with open(os.path.join(patch_path, "series"), "r") as patches_file: + for line in patches_file: + if line.strip() == "": + break + if line[0] == "#": + continue + self.patches.append(line.strip()) + for patch in self.get_certain_files(patch_path, ".patch", True): + shutil.copy2(patch, path) + for diff in self.get_certain_files(patch_path, ".diff", True): + shutil.copy2(diff, path) + + self.meta_data.update(dsc.meta_data) + self.meta_data['Files'].append(os.path.basename(self.dsc_name)) + self.control_data = control.package_data + self.meta_data['copyright'] = lic.license + + def get_certain_files(self, dir: str, pattern: str, get_sub: bool) -> List: + all_files = os.listdir(dir) + files = [os.path.join(dir, file) for file in all_files if file.endswith(pattern)] + # print(files) + if get_sub: + sub_dirs = [os.path.join(dir, sub_dir) for sub_dir in all_files if os.path.isdir(os.path.join(dir, sub_dir))] + for sub in sub_dirs: + files += self.get_certain_files(sub, pattern, True) + return files + +if __name__ == "__main__": + parser = DebParser('/home/young/rpmbuild/SOURCES/nghttp2/', '/home/young/rpmbuild/BUILD/nghttp2-1.43.0/') + print(parser.meta_data) + # print(parser.control_data) + for package_name, package_info_dir in parser.control_data.items(): + print(package_name) + for key, value in package_info_dir.items(): + print(f'{key}: {value}') diff --git a/tools/deb2rpm/scripts/get_dependency.py b/tools/deb2rpm/scripts/get_dependency.py new file mode 100644 index 0000000000000000000000000000000000000000..67471d56c41170baf54242d964a50366fa99551f --- /dev/null +++ b/tools/deb2rpm/scripts/get_dependency.py @@ -0,0 +1,139 @@ +import re +import os +import sys +import json + +from scripts.init_db import InitDB + +home = os.getenv("HOME") +mapping_path = f"{home}/deb2rpm/config/deb2rpm_mapping.json" + +package_info_keys = ["Package", "Provides", "PreDepends", "Depends", "Recommends", "Suggests", "Conflicts", "Obsoletes", "Source"] + +class DepParser: + def __init__(self, all_dep: str, map=False): + """ + 初始化依赖提取器 + @param all_dep 数据库中提取出的原始字符串 + """ + self.deps_and_version = [] + self.map = map + with open(mapping_path, 'r') as file: + self.mapping = json.load(file) + if all_dep != '': + for dep in all_dep.strip().split(", "): + self.process_each(dep) + + def process_each(self, deps: str) -> None: + """ + 依次处理每一段字符串 + @param deps 单个的依赖 + """ + # | 分隔的不同依赖选用第一个 + parts = deps.split('|') + dep = parts[0] + + # 忽略[]中的架构信息和<>中的其它信息 + dep_ignore1 = re.sub(r'\[[^\]]*\]', '', dep).strip() + dep_ignore0 = re.sub(r'<[^>]*>', '', dep_ignore1).strip() + + # 提取依赖名及版本信息 + dep_ignore = dep_ignore0.replace('(', '').replace(')', '') + dep_parts = dep_ignore.split() + + # 排除类似python3:any中的架构信息干扰 + names = dep_parts[0].split(":") + pkg_name = self.map_name(names[0]) + if pkg_name == "debhelper": + return + if len(dep_parts) == 3: + version = self.get_version(dep_parts[2]) + if dep_parts[1] == "<<": + self.deps_and_version.append(f"{pkg_name} < {version}") + elif dep_parts[1] == ">>": + self.deps_and_version.append(f"{pkg_name} > {version}") + else: + self.deps_and_version.append(f"{pkg_name} {dep_parts[1]} {version}") + else: + self.deps_and_version.append(pkg_name) + + def map_name(self, name: str) -> str: + """ + 包名映射 + @param name deb包名 + @return 对应rpm包名 + """ + if self.map and name in self.mapping.keys(): + return self.mapping[name] + # if name.endswith("-dev"): + # return f"{name}el" + return name + + def get_version(self, deb_version: str) -> str: + """ + 从deb版本号中提取version信息 + @param deb_version deb版本字符串 + """ + version_string = deb_version + epoch_pattern = r'^(\d+):' + release_pattern = r'(.*?)[-+~](.*)' + + match_epoch = re.match(epoch_pattern, deb_version) + if match_epoch: + epoch = match_epoch.group(1) + version_string = version_string[len(epoch)+1:] + match_release = re.match(release_pattern, version_string) + if match_release: + release = match_release.group(2).strip() + version_string = version_string[:-len(release)-1] + + return version_string + +class DepJson: + def __init__(self, repo_name: str, source_name: str, json_path=""): + """ + 初始化DepJson，获取所有信息 + @param repo_name 源名 + @param source_name 源码包名 + @param json_path 生成json文件所在文件夹路径(默认为当前路径) + """ + self.repo_name = repo_name + self.source_name = source_name + self.json_path = json_path + self.dic = {} + db = InitDB() + _, _, build_requires, _, packages, _ = db.get_source_info(self.repo_name, source_name) + + src = DepParser(build_requires, map=True) + build_requires_list = src.deps_and_version + self.dic["Name"] = source_name + self.dic["Build-Depends"] = build_requires_list + + self.dic["Packages"] = {} + for package in packages.strip().split(): + self.dic["Packages"][package]= {} + pac_info = db.get_package_info(self.repo_name, package) + if pac_info != None: + for i, value in enumerate(pac_info): + if i != 0 and i != 8 and i != 9 and value != "": + dep = DepParser(value, map=True) + self.dic["Packages"][package][package_info_keys[i]] = dep.deps_and_version + + # print(self.dic) + db.close_db() + + def write_into_json(self) -> None: + if self.json_path == "": + self.json_path = os.path.join(os.getcwd(), f"{self.source_name}.json") + + with open(self.json_path, 'w') as file: + json.dump(self.dic, file, indent=4, ensure_ascii=False) + + + +if __name__ == "__main__": + assert len(sys.argv) == 2 + target = sys.argv[1] + + dj = DepJson("main", target) + dj.write_into_json() diff --git a/tools/deb2rpm/scripts/get_files.py b/tools/deb2rpm/scripts/get_files.py new file mode 100644 index 0000000000000000000000000000000000000000..eaeb1eeeb7e1219e33fe442af3db36ed4b8c6454 --- /dev/null +++ b/tools/deb2rpm/scripts/get_files.py @@ -0,0 +1,453 @@ +import os +import sys +import re +import json +import logging +import shutil +<<<<<<< HEAD +======= +# import subprocess +>>>>>>> 6e53b74 (Change a method to extract files from deb) + +from scripts.init_db import InitDB + +from typing import List + +logger = logging.getLogger(__name__) + +home = os.getenv('HOME') + +json_path = f"{home}/deb2rpm/config/resources.json" +with open(json_path, 'r') as file: + json_data = json.load(file) +web_repo = json_data["repo_url"] + +def get_line_info(line: str) -> List: + """ + 从line中提取信息 + @param line 一行，格式为key: value + @return 返回[key, value] + """ + pattern = r'^(.*?):\s*(.*)' + match = re.search(pattern, line) + if match: + return match.group(1).strip(), match.group(2).strip() + return "", "" + +all_commands = [ + "debian/rules build", + "dh_testroot", + "dh_prep", + "dh_installdirs", + "dh_auto_install", + "dh_install", + "dh_installdocs", + "dh_installchangelogs", + "dh_installexamples", + "dh_installman", + "dh_installcatalogs", + "dh_installcron", + "dh_installdebconf", + "dh_installemacsen", + "dh_installifupdown", + "dh_installinfo", + "dh_installinit", + "dh_installsystemd", + "dh_installmenu", + "dh_installmime", + "dh_installmodules", + "dh_installlogcheck", + "dh_installlogrotate", + "dh_installpam", + "dh_installppp", + "dh_installudev", + "dh_installwm", + "dh_installxfonts", + "dh_bugfiles", + "dh_lintian", + # "dh_gconf", + "dh_icons", + "dh_installsystemduser", + "dh_perl", + "dh_usrlocal", + "dh_link", + "dh_compress", + "dh_fixperms", + "dh_missing", + "dh_dwz", + "dh_strip", +# "dh_makeshlibs", +# "dh_shlibdeps", +# "dh_installdeb", +# "dh_gencontrol", +# "dh_md5sums", +# "dh_builddeb", +] + +class DebFiles: + + def __init__(self, path: str, repo_name: str, source_name: str, download_debs: bool) -> None: + """ + @param path: deb源码包根目录 + 在path目录下执行操作获取信息 + """ + self.root_path = path + self.repo = repo_name + self.source = source_name + self.rules_targets = [] + self.files = {} + self.subdirs = {} + self.arch = {} + self.commands = all_commands # self.all_params = "" + os.chdir(path) + self._get_rules_targets() + # print(self.rules_targets) + self._refresh_commands() + self._get_packages() + if not download_debs: + self._exe_commands() + self._get_all_files() + else: + self._get_files_by_debs() + + # print(self.files) + # print(self.subdirs) + # print(self.arch) +<<<<<<< HEAD +======= + +>>>>>>> ba33252 (Hide part of output in get_files) + + def _get_rules_targets(self) -> None: + """ + 解析debian/rules文件，将所有target存入self.rules_targets + 并获取dh $@后的通用参数 + """ + pattern = r'(.*?):[^=](.*)' + pattern2 = r'dh\s+\$@\s+(.*)' + with open("debian/rules", 'r') as file: + for line in file: + if line.startswith('#'): + continue + match = re.search(pattern, line) + if match: + self.rules_targets.append(match.group(1).strip()) + + match2 = re.search(pattern2, line) + if match2 and match2.group(1).strip() != "": + logger.info(f"dh binary阶段忽略参数{match2.group(1).strip()}") + # self.all_params = match2.group(1).strip() + + def _refresh_commands(self) -> None: + """ + 根据rules文件中的target，修改commands的内容 + """ + to_be_changed = {} + for c in all_commands: + to_be_changed[c] = [] + + # 获取override字段 + pattern1 = r'override_(.*)' + pattern2 = r'(.*?)(?:-indep|-arch)' + for target in self.rules_targets: + match = re.search(pattern1, target) + if match: + match2 = re.search(pattern2, match.group(1).strip()) + if match.group(1).strip() in all_commands: + to_be_changed[match.group(1).strip()].append(f"debian/rules {target}") + elif match2 and match2.group(1).strip() in all_commands: + to_be_changed[match2.group(1).strip()].append(f"debian/rules {target}") + + # 更新命令 + for key, value_list in to_be_changed.items(): + if len(value_list) != 0: + i = self.commands.index(key) + self.commands = self.commands[:i] + value_list + self.commands[i+1:] + # print(self.commands) + # 有关清理任务 + if "override_dh_auto_clean" in self.rules_targets: + self.commands.insert(0, "debian/rules override_dh_auto_clean") + else: + self.commands.insert(0, "debian/rules clean") + + def _exe_commands(self) -> None: + for command in self.commands: + logger.info(f"执行命令 fakeroot {command}") + res = os.system("fakeroot " + command) + if res != 0: + logger.error(f"执行{command}失败！") + sys.exit() + + def _get_packages(self) -> None: + """ + 从debian/control文件中获取所有子包名 + """ + package = "" + with open("debian/control", 'r') as file: + for line in file: + if line.startswith('#'): + continue + key, value = get_line_info(line) + if key == "Package": + package = value + self.files[package] = [] + self.subdirs[package] = [] + self.arch[package] = "" + elif key == "Architecture": + if value == "any": + self.arch[package] = "amd64" + else: + self.arch[package] = value + + def _get_all_files(self) -> None: + """ + 将子包所有的文件添加到files字典中 + """ + for p in self.files.keys(): + path = os.path.join(self.root_path, "debian", p) + self.files[p], self.subdirs[p] = self._get_all_files_under_dir(path) + + def _get_all_files_under_dir(self, path: str) -> List: + """ + @param path 目标文件夹 + @return 目标文件夹下所有文件的列表 + """ + all_files = [] + empty_dir = [] + for root, subdirs, files in os.walk(path): + if 'DEBIAN' in subdirs: + subdirs.remove("DEBIAN") + if len(files) == 0 and len(subdirs) == 0: + empty_dir.append(os.path.relpath(root, path)) + for file in files: + abs_path = os.path.join(root, file) + all_files.append(os.path.relpath(abs_path, path)) + + return all_files, empty_dir + + def _get_files_by_debs(self): + db = InitDB() +<<<<<<< HEAD +<<<<<<< HEAD + _, _, _, _, packages, _ = db.get_source_info(self.repo, self.source) + cur_dir = os.getcwd() + os.makedirs("debs", exist_ok=True) + os.chdir("debs") +======= + _, version, _, dir, packages, _ = db.get_source_info(self.repo, self.source) + version = version[version.find(':')+1:] +<<<<<<< HEAD + os.system("mkdir debs && cd debs") +>>>>>>> ba33252 (Hide part of output in get_files) +======= + cur_dir = os.getcwd() + os.makedirs("debs", exist_ok=True) + os.chdir("debs") + # os.system("mkdir debs && cd debs") +>>>>>>> 5d59dad (Fix bugs in getting files) + for package in packages.strip().split(): + pac_info = db.get_package_info(self.repo, package) + if not pac_info: + logger.error(f"Didn't find deb of {package}") + sys.exit() +======= + _, _, _, _, packages, _ = db.get_source_info(self.repo, self.source) + cur_dir = os.getcwd() + os.makedirs("debs", exist_ok=True) + os.chdir("debs") + for package in packages.strip().split(): + pac_info = db.get_package_info(self.repo, package) +<<<<<<< HEAD +>>>>>>> 8dba3e5 (Add path of deb to database) +======= + if not pac_info: + logger.error(f"Didn't find deb of {package}") + sys.exit() +>>>>>>> 4c88c73 (Exit when hitting a Not Found of deb) + deb_dir_and_name = pac_info[-1] + deb_name = list(deb_dir_and_name.split('/'))[-1] + web_path = f"{web_repo}/{deb_dir_and_name}" + # os.system("pwd") + res = os.system(f"wget {web_path}") + if res != 0: + logger.error(f"未找到{web_path}！") +<<<<<<< HEAD +<<<<<<< HEAD + sys.exit() + else: + self.files[package], self.subdirs[package] = self._files_in_deb(deb_name) + # os.remove(f"{deb_name}") + + os.chdir(cur_dir) +======= + if self.arch[package] == "all": + alter_deb_name = f"{package}_{version}_amd64.deb" + res2 = os.system(f"wget {web_repo}/{dir}/{alter_deb_name}") + if res2 != 0: + logger.error(f"未找到 {web_repo}/{dir}/{alter_deb_name}") + sys.exit() + else: + self.files[package], self.subdirs[package] = self._files_in_deb(alter_deb_name) + # os.remove(f"{alter_deb_name}") + else: + sys.exit() +======= + sys.exit() +>>>>>>> 8dba3e5 (Add path of deb to database) + else: + self.files[package], self.subdirs[package] = self._files_in_deb(deb_name) + # os.remove(f"{deb_name}") +>>>>>>> 5d59dad (Fix bugs in getting files) + + os.chdir(cur_dir) + + def _files_in_deb(self, path) -> List: + os.makedirs(f'/tmp/{self.source}', exist_ok=True) + name = os.path.basename(path) + if not os.path.exists(f'/tmp/{self.source}/{name}'): + shutil.move(path, f'/tmp/{self.source}') + cur_dir = os.getcwd() + os.chdir(f'/tmp/{self.source}') + ret = os.popen(f'ar x {name}') + ret.read() + ret = os.popen(f'zstd -d -c data.tar.zst | tar -vxf - ') +<<<<<<< HEAD +<<<<<<< HEAD +<<<<<<< HEAD + fileTree = FileNode("") +<<<<<<< HEAD + + for line in ret.read().split('\n'): + parts = line.split('/') + path_parts = [part for part in parts if part and part != '.'] +======= + # print(output) + for line in output.strip().split('\n'): + path = list(line.split())[5][1:] + # print(path) + parts = path.split('/') + path_parts = [part for part in parts if part] + # print(path_parts) +>>>>>>> ba33252 (Hide part of output in get_files) + current_node = fileTree + for i, path_part in enumerate(path_parts): + if i != len(path_parts) - 1: + if path_part not in current_node.children.keys(): + current_node.children[path_part] = FileNode(path_part) + current_node = current_node.children[path_part] + else: + dir = path.endswith('/') + if path_part not in current_node.children.keys(): + current_node.children[path_part] = FileNode(path_part, dir) + os.chdir(cur_dir) + return fileTree.get_files_and_dirs() + + +class FileNode: + def __init__(self, name: str, dir=True): + self.name = name + self.children = {} + self.dir = dir + + def get_files_and_dirs(self) -> List: + if len(self.children) == 0: + if self.dir: + return [], [f"{self.name}"] + else: + return [f"{self.name}"], [] + +======= +>>>>>>> 6e53b74 (Change a method to extract files from deb) + files = [] + dirs = [] + for line in ret.read().split('\n'): + parts = line.split('/') + path_parts = [part for part in parts if part and part != '.'] + if not path_parts: + continue + abs_path = f'/tmp/{self.source}' + if os.path.isdir(abs_path): + dirs.append(abs_path) + else: + files.append(abs_path) + os.chdir(cur_dir) + return files, dirs +======= + # files = [] + # dirs = [] + # for line in ret.read().split('\n'): + # parts = line.split('/') + # path_parts = [part for part in parts if part and part != '.'] + # if not path_parts: + # continue + # abs_path = f'/tmp/{self.source}' + # if os.path.isdir(abs_path): + # dirs.append(abs_path) + # else: + # files.append(abs_path) + # os.chdir(cur_dir) + # return files, dirs +>>>>>>> 5d59dad (Fix bugs in getting files) + + # try: + # output = subprocess.check_output(['dpkg', '-c', path], universal_newlines=True) + # except Exception as e: + # print(f"dpkg -c Error: {e}") + # +======= +>>>>>>> 4c88c73 (Exit when hitting a Not Found of deb) + fileTree = FileNode("") + + for line in ret.read().split('\n'): + parts = line.split('/') + path_parts = [part for part in parts if part and part != '.'] + current_node = fileTree + for i, path_part in enumerate(path_parts): + if i != len(path_parts) - 1: + if path_part not in current_node.children.keys(): + current_node.children[path_part] = FileNode(path_part) + current_node = current_node.children[path_part] + else: + dir = path.endswith('/') + if path_part not in current_node.children.keys(): + current_node.children[path_part] = FileNode(path_part, dir) + os.chdir(cur_dir) + return fileTree.get_files_and_dirs() + + +class FileNode: + def __init__(self, name: str, dir=True): + self.name = name + self.children = {} + self.dir = dir + + def get_files_and_dirs(self) -> List: + if len(self.children) == 0: + if self.dir: + return [], [f"{self.name}"] + else: + return [f"{self.name}"], [] + + files = [] + dirs = [] + for child in self.children.values(): + files += [f"{self.name}/{file}" for file in list(child.get_files_and_dirs())[0]] + dirs += [f"{self.name}/{dir}" for dir in list(child.get_files_and_dirs())[1]] + + return files, dirs + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Provide one directory!") + sys.exit() + + root_dir = os.getcwd() + package_dir = os.path.join(root_dir, sys.argv[1]) + all_subdir = [os.path.join(package_dir, x) for x in os.listdir(package_dir) + if os.path.isdir(os.path.join(package_dir, x))] + + if len(all_subdir) == 1: + DebFiles(all_subdir[0], "main", sys.argv[1], True) + # os.chdir("dh-python/dh-python-5.20220403") + # os.system("fakeroot debian/rules clean") + # os.system("fakeroot debian/rules build") diff --git a/tools/deb2rpm/scripts/get_repos.py b/tools/deb2rpm/scripts/get_repos.py new file mode 100644 index 0000000000000000000000000000000000000000..d6b11054e4d8d8e309c57dc15c7dad27863b741f --- /dev/null +++ b/tools/deb2rpm/scripts/get_repos.py @@ -0,0 +1,57 @@ +import os +import sys +import json + +home = os.getenv("HOME") + +cates = ['', '-updates', '-backports', '-security', '-proposed'] +repos = ['main', 'multiverse', 'restricted', 'universe'] + +def create_dic(name: str): + os.system(f"rm -rf {home}/Package20 && mkdir {home}/Package20") + os.system(f"cd {home}/Package20 && mkdir " + ' '.join([f"{name}"+i for i in cates])) + os.system(f"rm -rf {home}/Source20 && mkdir {home}/Source20") + os.system(f"cd {home}/Source20 && mkdir " + ' '.join([f"{name}"+i for i in cates])) + +def get_source_package(name: str, url: str, cate: str, repo: str): + source_url = url + '/dists/' + f'{name+cate}/{repo}/source/Sources.gz' + package_url = url + '/dists/' + f'{name+cate}/{repo}/binary-amd64/Packages.gz' + + os.system(f"cd {home}/Source20/{name+cate} && wget {source_url}") + os.system(f"gzip -d {home}/Source20/{name+cate}/Sources.gz") + os.system(f"mv {home}/Source20/{name+cate}/Sources {home}/Source20/{name+cate}/{'Sources-'+repo}") + + os.system(f"cd {home}/Package20/{name+cate} && wget {package_url}") + os.system(f"gzip -d {home}/Package20/{name+cate}/Packages.gz") + os.system(f"mv {home}/Package20/{name+cate}/Packages {home}/Package20/{name+cate}/{'Packages-'+repo}") + +if __name__ == "__main__": + assert len(sys.argv) == 3 + version_name = sys.argv[1] + url = sys.argv[2] + create_dic(version_name) + for cate in cates: + for repo in repos: + get_source_package(version_name, url, cate, repo) + + json_data = {} + json_data["refresh"] = False + json_data["repo_url"] = url + json_data["main"] = {} + for cate in cates: + vn = version_name + cate + if cate == '': + cate = '-main' + + cate_name = cate[1:] + json_data[cate_name] = {} + json_data[cate_name]["enable"] = True + json_data[cate_name]["Sources"] = {} + for repo in repos: + json_data[cate_name]["Sources"][repo] = f"{home}/Source20/{vn}/Sources-{repo}" + json_data[cate_name]["Packages"] = {} + for repo in repos: + json_data[cate_name]["Packages"][repo] = f"{home}/Package20/{vn}/Packages-{repo}" + + with open(f"{home}/deb2rpm/config/resources.json", 'w') as file: + json.dump(json_data, file, indent=4, ensure_ascii=False) diff --git a/tools/deb2rpm/scripts/init_db.py b/tools/deb2rpm/scripts/init_db.py new file mode 100644 index 0000000000000000000000000000000000000000..72fe6652dbee25bf4e708c0b6314740bebd53587 --- /dev/null +++ b/tools/deb2rpm/scripts/init_db.py @@ -0,0 +1,138 @@ +import os +import sys +import json +import sqlite3 +import logging + +from typing import List + +project_path = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +sys.path.append(project_path) + +from scripts.sqlite_database import BaseDB, SourceDB, PackageDB + +home = os.getenv("HOME") +logging.basicConfig(level=logging.INFO, + format='%(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +json_file = f"{home}/deb2rpm/config/resources.json" + +repos = ['main', 'updates', 'backports', 'security', 'proposed'] + +paths_table_name = "paths" +sources_table_names = ['SourcesMain', 'SourcesMultiverse', 'SourcesRestricted', 'SourcesUniverse'] +packages_table_names = ['PackagesMain', 'PackagesMultiverse', 'PackagesRestricted', 'PackagesUniverse'] +resources_table_names = sources_table_names + packages_table_names + +paths_table_keys = ['source_or_package', 'main', 'multiverse', 'restricted', 'universe'] +paths_table_types = ['TinyText', 'TinyText', 'TinyText', 'TinyText', 'TinyText'] + +class InitDB(BaseDB): + def __init__(self): + super().__init__() + with open(json_file, 'r') as file: + json_data = json.load(file) + refresh = json_data['refresh'] + self.enables = [json_data[repo]['enable'] for repo in repos] + self.sources_path = {} + self.packages_path = {} + + for repo in repos: + self.sources_path[repo] = json_data[repo]['Sources'] + self.packages_path[repo] = json_data[repo]['Packages'] + + for repo in repos: + if refresh or not super().table_exists(repo + paths_table_name): + for name in resources_table_names: + super().drop_table(repo + name) + self.initial_SP_tables(repo) + else: + origin_sources = super().select_data(repo + paths_table_name, 'source_or_package', 'Sources') + cnt = 1 + for name, path in zip(sources_table_names, self.sources_path[repo].values()): + if origin_sources[cnt] != path: + super().drop_table(repo + name) + SourceDB(repo + name, path) + logger.info(f"{repo}{name}表完成更新！") + cnt += 1 + + origin_packages = super().select_data(repo + paths_table_name, 'source_or_package', 'Packages') + cnt = 1 + for name, path in zip(packages_table_names, self.packages_path[repo].values()): + if origin_packages[cnt] != path: + super().drop_table(repo + name) + PackageDB(repo + name, path) + logger.info(f"{repo}{name}表完成更新！") + cnt += 1 + + self.initial_paths_table(repo) + + def initial_paths_table(self, repo_name: str): + name = repo_name + paths_table_name + super().drop_table(name) + super().create_table(name, paths_table_keys, paths_table_types) + super().insert_data(name, ['Sources'] + list(self.sources_path[repo_name].values())) + super().insert_data(name, ['Packages'] + list(self.packages_path[repo_name].values())) + + def initial_SP_tables(self, repo: str): + for name, source in zip(sources_table_names, self.sources_path[repo].values()): + SourceDB(repo + name, source) + logger.info(f"{repo}{name}表初始化完成!") + for name, package in zip(packages_table_names, self.packages_path[repo].values()): + PackageDB(repo + name, package) + logger.info(f"{repo}{name}表初始化完成!") + + def get_source_info(self, repo_name:str, source_name: str) -> List: + """ + 获取数据库中Source信息的接口 + @param repo_name 库名 + @param source_name source名 + """ + query = f''' + SELECT * FROM {repo_name + sources_table_names[0]} WHERE Package = '{source_name}' + UNION + SELECT * FROM {repo_name + sources_table_names[1]} WHERE Package = '{source_name}' + UNION + SELECT * FROM {repo_name + sources_table_names[2]} WHERE Package = '{source_name}' + UNION + SELECT * FROM {repo_name + sources_table_names[3]} WHERE Package = '{source_name}' + ''' + self.cursor.execute(query) + return self.cursor.fetchone() + + def get_package_info(self, repo_name: str, package_name: str) -> List: + """ + 获取数据库中Package信息的接口 + @param repo_name 库名 + @param package_name package名 + """ + query = f''' + SELECT * FROM {repo_name + packages_table_names[0]} WHERE Package = '{package_name}' + UNION + SELECT * FROM {repo_name + packages_table_names[1]} WHERE Package = '{package_name}' + UNION + SELECT * FROM {repo_name + packages_table_names[2]} WHERE Package = '{package_name}' + UNION + SELECT * FROM {repo_name + packages_table_names[3]} WHERE Package = '{package_name}' + ''' + self.cursor.execute(query) + return self.cursor.fetchone() + + def close_db(self): + self.conn.close() + +if __name__ == "__main__": + assert len(sys.argv) == 2 + target = sys.argv[1] + db = InitDB() + if target == "deb2rpm_test": + print("Initialize the database successfully!") + else: + source_info = db.get_source_info("main", target) + package_info = db.get_package_info("main", target) + # print(source_info[0]) + print(source_info) + # print(package_info[0]) + print(package_info) + db.close_db() diff --git a/tools/deb2rpm/scripts/json2spec.py b/tools/deb2rpm/scripts/json2spec.py new file mode 100644 index 0000000000000000000000000000000000000000..038b46fa1e11a4fc3737420809df2afcbc419558 --- /dev/null +++ b/tools/deb2rpm/scripts/json2spec.py @@ -0,0 +1,41 @@ +import os +import json + +class Json2Spec: + def __init__(self, spec_path: str, json_path: str): + """ + 初始化Json2Spec + @param spec_path spec文件路径 + @param json_path json文件路径 + """ + with open(json_path, 'r') as json_file: + self.dic = json.load(json_file) + + with open(spec_path, 'r') as file: + lines = file.readlines() + + with open(spec_path, 'w') as spec_file: + for line in lines: + if line.strip() == "# BuildRequires": + for br in self.dic["Build-Depends"]: + spec_file.write(f"BuildRequires: {br}\n") + if self.dic["Name"] in self.dic["Packages"].keys(): + self.write_package_deps(self.dic["Name"], spec_file) + elif line.startswith("# dep"): + parts = line.strip().split() + package_name = parts[-1] + self.write_package_deps(package_name, spec_file) + else: + spec_file.write(line) + + def write_package_deps(self, package_name: str, spec_file): + for key, dep_list in self.dic["Packages"][package_name].items(): + for dep in dep_list: + if key == 'PreDepends' or key == 'Depends': + spec_file.write(f"Requires: {dep}\n") + else: + spec_file.write(f"{key}: {dep}\n") + +if __name__ == "__main__": + home = os.getenv("HOME") + Json2Spec(f"{home}/rpmbuild/SPECS/dh-python-5.20220403.spec", f"{home}/rpmbuild/SPECS/dh-python.json") diff --git a/tools/deb2rpm/scripts/recursive_depends.py b/tools/deb2rpm/scripts/recursive_depends.py new file mode 100644 index 0000000000000000000000000000000000000000..4b570ee671757bcb1a3ea0c15c918d15a1f30f9c --- /dev/null +++ b/tools/deb2rpm/scripts/recursive_depends.py @@ -0,0 +1,252 @@ +import os +import sys +import json +import logging +import networkx as nx + +from typing import List + +from scripts.init_db import InitDB +from scripts.get_dependency import DepParser + +home = os.getenv("HOME") +mapping_path = f"{home}/deb2rpm/config/deb2rpm_mapping.json" + +logger = logging.getLogger(__name__) + +class DeepDep: + def __init__(self, repo_name: str, target: str, json_path=""): + """ + 初始化DeepDep类，递归获取对应包的所有编译和运行依赖 + @param repo_name 源名 + @param target 目标包 + """ + self.target = target + self.repo_name = repo_name + self.json_path = json_path + self.data_dic = {} + self.package_order = [] + self.source_order = [] + self.source_and_package = {} + self.dep_tree = DepTree(target) + self.exists = [] + self.db = InitDB() + + with open(mapping_path, 'r') as file: + self.mapping = json.load(file) + + if self.target not in self.mapping.keys(): + self.recursive_deps(target, 0) + + self.install_order() + self.get_source_order() + + + def recursive_deps(self, package_name: str, depth: int): + """ + 递归获取依赖包的依赖 + @param package_name 包名 + @param depth 依赖深度(目标为0) + """ + # print(package_name) + if package_name in self.data_dic.keys(): + if depth > self.data_dic[package_name]["Depth"]: + self.data_dic[package_name]["Depth"] = depth + return + + # print(f"正在处理 {package_name}") + logger.info(f"正在解析 {package_name}") + self.data_dic[package_name] = {} + self.data_dic[package_name]["Depth"] = depth + self.data_dic[package_name]["Build-Depends"] = [] + self.data_dic[package_name]["Pre-Depends"] = [] + self.data_dic[package_name]["Depends"] = [] + + src = self.db.get_source_info(self.repo_name, package_name) + if (src != None): + src_build_deps = DepParser(src[2]) + if len(src_build_deps.deps_and_version) != 0: + for bd in src_build_deps.deps_and_version: + parts = bd.split() + bd_name = parts[0] + self.data_dic[package_name]["Build-Depends"].append(bd) + if bd_name not in self.mapping.keys(): + self.dep_tree.add_edge(package_name, bd_name) + self.recursive_deps(bd_name, depth + 1) + elif self.mapping[bd_name] not in self.exists: + self.exists.append(self.mapping[bd_name]) + + pac_info = self.db.get_package_info(self.repo_name, package_name) + if (pac_info != None): + pre_depends = pac_info[2] + depends = pac_info[3] + + if (pre_depends != ""): + pre_depends_parser = DepParser(pre_depends) + for pd in pre_depends_parser.deps_and_version: + parts = pd.split() + pd_name = parts[0] + self.data_dic[package_name]["Pre-Depends"].append(pd) + if pd_name not in self.mapping.keys(): + self.dep_tree.add_edge(package_name, pd_name) + self.recursive_deps(pd_name, depth + 1) + elif self.mapping[pd_name] not in self.exists: + self.exists.append(self.mapping[pd_name]) + + if (depends != ""): + depends_parser = DepParser(depends) + for d in depends_parser.deps_and_version: + parts = d.split() + d_name = parts[0] + self.data_dic[package_name]["Depends"].append(d) + if d_name not in self.mapping.keys(): + self.dep_tree.add_edge(package_name, d_name) + self.recursive_deps(d_name, depth + 1) + elif self.mapping[d_name] not in self.exists: + self.exists.append(self.mapping[d_name]) + + def __str__(self): + return json.dumps(self.data_dic, indent=4, ensure_ascii=False) + + def write_into_json(self) -> None: + if self.json_path == "": + return + + with open(self.json_path, 'w') as file: + json.dump(self.data_dic, file, indent=4, ensure_ascii=False) + + def install_order(self) -> None: + """ + 拓扑排序给出的安装顺序 + """ + def get_items(items: List): + items_list = [] + for i in items: + if i == self.target: + continue + if i not in self.dep_tree.all_cycles.keys(): + items_list.append(i) + else: + items_list += get_items(self.dep_tree.all_cycles[i]) + + return items_list + + if self.target in self.mapping.keys(): + self.package_order = [self.mapping[self.target]] + + self.package_order = get_items(self.dep_tree.reversed_topological_sort()) + [self.target] + + def get_source_order(self): + """ + 根据安装顺序给出需要编译的源码包顺序及对应关系 + """ + for i in self.package_order[:-1]: + pac_info = self.db.get_package_info(self.repo_name, i) + if pac_info != None: + source_name = pac_info[-1] + if source_name not in self.source_and_package.keys(): + self.source_and_package[source_name] = [i] + else: + self.source_and_package[source_name].append(i) + + if source_name not in self.source_order: + self.source_order.append(source_name) + + +class DepTree: + def __init__(self, target:str, delete_cycles=True): + self.G = nx.DiGraph() + self.G.add_node(target) + self.delete_cycles = delete_cycles + self.cycle_cnt = 0 + self.node2cycle = {} + self.all_cycles = {} + + def add_edge(self, source, to): + while source in self.node2cycle.keys(): + source = self.node2cycle[source] + source_cycle = source + + while to in self.node2cycle.keys(): + to = self.node2cycle[to] + to_cycle = to + self.G.add_edge(source_cycle, to_cycle) + if self.delete_cycles: + self.process_cycles(source_cycle) + + def reversed_topological_sort(self) -> List: + return list(reversed(list(nx.topological_sort(self.G)))) + + def process_cycles(self, source) -> None: + """ + 将存在的环用新的节点替代，直至不存在环 + """ + try: + self.G.remove_edges_from(nx.selfloop_edges(self.G)) + cycles = list(nx.find_cycle(self.G, source, orientation='original')) + # cycles = [cyc for cyc in cycles_list if len(set(cyc[:-1])) > 1] + while (len(cycles) != 0): + new_node = f"cycle{self.cycle_cnt}" + self.all_cycles[new_node] = [] + cycle = [e[0] for e in cycles] + # print(f"处理环cycle{self.cycle_cnt}: {cycle}") + + # G_copy = self.G.copy() + + # 创建新的节点 + self.G.add_node(new_node) + + # 重定向所有指向环中节点的边到新节点 + for source in self.G.nodes(): + if source not in cycle: + for target in cycle: + if self.G.has_edge(source, target): + # self.G.remove_edge(source, target) + self.G.add_edge(source, new_node) + + # 重定向所有从环中节点发出的边到新节点 + for node in cycle: + neighbors = self.G.neighbors(node) + for target in neighbors: + if target not in cycle: + # self.G.remove_edge(node, target) + self.G.add_edge(new_node, target) + + for node in cycle: + # 删除原节点并记录 + self.node2cycle[node] = new_node + if node in self.all_cycles.keys(): + for n in self.all_cycles[node]: + self.all_cycles[new_node].append(n) + else: + self.all_cycles[new_node].append(node) + if node in self.all_cycles.keys(): + for target in self.all_cycles[node]: + self.node2cycle[target] = new_node + self.G.remove_node(node) + + self.cycle_cnt += 1 + self.G.remove_edges_from(nx.selfloop_edges(self.G)) + cycles = list(nx.find_cycle(self.G, new_node, orientation='original')) + + except nx.exception.NetworkXNoCycle: + return + + +if __name__ == "__main__": + assert len(sys.argv) == 2 + target = sys.argv[1] + + dd = DeepDep("main", target) + print(dd) + print(dd.exists) + print(dd.package_order) + print(dd.source_order) + # print(dd.dep_tree.all_cycles) + # print(dd.install_order()) + # for i in dd.install_order(): + # if i not in dd.dep_tree.all_cycles.keys(): + # print(i, end=" ") + # else: + # print(dd.dep_tree.all_cycles[i]) + diff --git a/tools/deb2rpm/scripts/spec.py b/tools/deb2rpm/scripts/spec.py new file mode 100644 index 0000000000000000000000000000000000000000..833fc1439c411bfbe909aa8b7e89b7b4baaa7502 --- /dev/null +++ b/tools/deb2rpm/scripts/spec.py @@ -0,0 +1,314 @@ +import os +import sys +import re +import json +import logging +import shutil +import platform + +from datetime import datetime + +from scripts.deb_parser import DebParser +from scripts.get_files import DebFiles +from scripts.init_db import InitDB + +logger = logging.getLogger(__name__) + +home = os.getenv('HOME') + +json_path = f"{home}/deb2rpm/config/resources.json" +with open(json_path, 'r') as file: + json_data = json.load(file) +web_repo = json_data["repo_url"] + +info_map = { + 'Source': 'Name', + 'Homepage': 'URL', + 'Files': 'Source', + 'copyright': 'License', +} + +spec_head_data = ['Name', 'Epoch', 'Version', 'Release', 'License', 'URL'] + +def copy_directory_contents(source, destination): + for item in os.listdir(source): + if item == 'DEBIAN': + continue + + source_item = os.path.join(source, item) + destination_item = os.path.join(destination, item) + + if os.path.isdir(source_item): + if os.path.exists(destination_item): + if os.path.isdir(destination_item): + copy_directory_contents(source_item, destination_item) + else: + shutil.copytree(source_item, destination_item, symlinks=True) + elif os.path.isfile(source_item): + shutil.copy2(source_item, destination_item) + elif os.path.islink(source_item): + target_path = os.readlink(source_item) + os.symlink(target_path, destination_item) + +class Spec: + def __init__(self, path: str, build_path: str, repo_name: str, source_name: str, download_debs: bool): + """ + 初始化存储信息的Spec类 + @param path 执行apt source的目录(与DebParser相同) + @param build_path 源码包根目录(与DebFiles相同) + """ + self.meta_data = {} + self.files = {} + self.subdirs = {} + self.packages = {} + self.patches = [] + self.build_commands = [] + + # 解析dsc和control文件获取元数据信息 + deb_parser = DebParser(path, build_path) + for key, value in info_map.items(): + if key in deb_parser.meta_data.keys(): + self.meta_data[value] = deb_parser.meta_data[key] + + self.get_version(deb_parser.meta_data["Version"]) + + self.patches = deb_parser.patches + self.packages = deb_parser.control_data + + # 完成构建并获取文件信息 + deb_files = DebFiles(build_path, repo_name, source_name, download_debs) + self.files = deb_files.files + self.subdirs = deb_files.subdirs + self.build_commands = deb_files.commands + + def get_version(self, deb_version: str) -> None: + """ + 获取Epoch, Version, Release信息 + @param deb_version deb版本字符串 + """ + version_string = deb_version + epoch_pattern = r'^(\d+):' + release_pattern = r'(.*?)-(.*)' + + match_epoch = re.match(epoch_pattern, deb_version) + if match_epoch: + epoch = match_epoch.group(1) + self.meta_data['Epoch'] = epoch + version_string = version_string[len(epoch)+1:] + match_release = re.match(release_pattern, version_string) + if match_release: + release = match_release.group(2).strip() + version_string = version_string[:-len(release)-1] + self.meta_data["Release"] = '1' + self.meta_data["Version"] = version_string + +class SpecGenarator: + def __init__(self, repo_name: str, + target_name: str, + download_debs: bool, + rpm_source_path=f'{home}/rpmbuild/SOURCES', + target_path=f'{home}/rpmbuild/SPECS', + buildroot_path=f'{home}/rpmbuild/BUILDROOT'): + """ + 生成spec文件 + @param source_path 源码压缩包路径(与Spec类path相同) + @param repo_name 源名 + @param target_path 新建文件所属文件夹 + @param build_path 构建目录(与Spec类build_path相同) + @param buildroot_path buildroot路径 + """ + self.rpm_source_path = rpm_source_path + self.source_path = os.path.join(rpm_source_path, target_name) + self.target_path = target_path + self.buildroot_path = buildroot_path + self.db = InitDB() + + source_name, version, build_requires, dir_path, _, source_files = self.db.get_source_info(repo_name, target_name) + if os.path.exists(self.source_path): + shutil.rmtree(self.source_path) + logger.info(f"创建目录{self.source_path}") + os.mkdir(self.source_path) + + # 获取源文件 + os.chdir(self.source_path) + for file in source_files.split(): + logger.info(f"获取文件{file}") + os.system(f"wget {web_repo}/{dir_path}/{file}") + + # 生成build目录 + self.file_parts = source_files.split() + logger.info("生成build目录") + os.system(f"dpkg-source -x --no-check {self.file_parts[0]}") + + # 获取目录路径 + subdir = [os.path.join(self.source_path, x) for x in os.listdir(self.source_path) + if os.path.isdir(os.path.join(self.source_path, x))] + assert len(subdir) == 1, "源码包错误！" + + spec = Spec(self.source_path, subdir[0], repo_name, source_name, download_debs) + source_name = spec.meta_data['Name'] + spec_name = f"{source_name}-{spec.meta_data['Version']}.spec" + self.spec_file_path = os.path.join(target_path, spec_name) + with open(self.spec_file_path, 'w') as spec_file: + # 开头的元数据部分 + for key in spec_head_data: + if key in spec.meta_data.keys(): + spec_file.write(f'{key}: {spec.meta_data[key]}\n') + + if source_name in spec.packages.keys(): + spec_file.write(f"Summary: {spec.packages[source_name]['Summary']}\n") + else: + spec_file.write(f"Summary: the {source_name} source package\n") + + spec_file.write('\n') + spec_file.write('# BuildRequires\n') + + # Source和Patch字段 + spec_file.write('\n') + source_cnt = 0 + for source in spec.meta_data['Source']: + spec_file.write(f'Source{source_cnt}: {source}\n') + source_cnt += 1 + + patch_cnt = 0 + if len(spec.patches) != 0: + spec_file.write('\n') + for patch in spec.patches: + spec_file.write(f'Patch{patch_cnt}: {patch}\n') + patch_cnt += 1 + + spec_file.write('\n') + spec_file.write('%description\n') + if source_name in spec.packages.keys(): + spec_file.write(f"{spec.packages[source_name]['description']}\n") + else: + spec_file.write(f"the description of {source_name} is temporarily missing\n") + + # 子包元数据 + spec_file.write('\n') + for package_name, package_info_dir in spec.packages.items(): + if package_name == source_name: + continue + spec_file.write(f'%package {package_name}\n') + spec_file.write(f"Summary: {package_info_dir['Summary']}\n") + spec_file.write(f"# deps of {package_name}\n") + + spec_file.write(f'\n') + spec_file.write(f'%description {package_name}\n') + spec_file.write(f"{package_info_dir['description']}\n") + + spec_file.write('\n') + self.write_prep_commands(spec, spec_file, source_cnt, patch_cnt) + spec_file.write('\n') + self.write_build_commands(spec, spec_file) + spec_file.write('\n') + self.write_install_commands(spec, spec_file) + + if source_name in spec.files.keys(): + spec_file.write('\n') + spec_file.write('%files\n') + if len(spec.subdirs[source_name]) != 0: + for dir in spec.subdirs[source_name]: + spec_file.write(f'%dir /{dir}\n') + for file in spec.files[source_name]: + spec_file.write(f'/{file}\n') + + # 子包files + spec_file.write('\n') + for package_name, files in spec.files.items(): + if package_name == source_name: + continue + spec_file.write(f'%files {package_name}\n') + if len(spec.subdirs[package_name]) != 0: + for dir in spec.subdirs[package_name]: + spec_file.write(f'%dir /{dir}\n') + for file in files: + spec_file.write(f'/{file}\n') + spec_file.write('\n') + + # changelog + spec_file.write('\n') + spec_file.write('%changelog\n') + spec_file.write(f"* {datetime.now().strftime('%a %b %d %Y')} OpenEuler {spec.meta_data['Version']}-{spec.meta_data['Release']}\n") + spec_file.write(f'- Build {source_name} from deb to rpm\n') + + self.copy2source() + # shutil.rmtree(self.source_path) + + def write_prep_commands(self, spec: Spec, spec_file, source_cnt, patch_cnt): + """ + 填写%prep字段 + @param spec spec对象 + @param spec_file spec文件对象 + @param source_cnt Source文件个数 + @param patch_cnt Patch文件个数 + """ + assert source_cnt >= 2, "文件数应不少于2" + spec_file.write('%prep\n') + + spec_file.write('rm -rf %{_builddir}/%{name}-%{version}\n') + spec_file.write('mkdir %{_builddir}/%{name}-%{version}\n') + spec_file.write(f"tar -{self.extract_string(spec.meta_data['Source'][0])} %{{SOURCE0}} -C %{{_builddir}}/%{{name}}-%{{version}} --strip-components=1\n") + for i in range(1, source_cnt): + if spec.meta_data['Source'][i].endswith(".tar.gz") or spec.meta_data['Source'][i].endswith(".tar.xz") or spec.meta_data["Source"][i].endswith(".tar.bz2"): + spec_file.write(f"tar -{self.extract_string(spec.meta_data['Source'][i])} %{{SOURCE{i}}} -C %{{_builddir}}/%{{name}}-%{{version}}\n") + + spec_file.write('\n') + if patch_cnt > 0: + spec_file.write('cd %{_builddir}/%{name}-%{version}\n') + for i in range(patch_cnt): + spec_file.write(f"%patch{i} -p1\n") + + def extract_string(self, file_name: str) -> str: + """ + 根据文件名返回tar解压命令参数 + """ + if file_name.endswith(".tar.gz"): + return "xzf" + if file_name.endswith(".tar.xz"): + return "xJf" + if file_name.endswith(".tar.bz2"): + return "xjf" + + + def write_build_commands(self, spec: Spec, spec_file): + """ + 填写%build字段 + @param spec spec对象 + @param spec_file spec文件对象 + """ + spec_file.write('%build\n') + spec_file.write('cd %{name}-%{version}\n') + for command in spec.build_commands: + spec_file.write(f'fakeroot {command}\n') + + def write_install_commands(self, spec: Spec, spec_file): + """ + 填写%install字段 + @param spec spec对象 + @param spec_file spec文件对象 + """ + spec_file.write('%install\n') + spec_file.write('%define _unpackaged_files_terminate_build 0\n') + spec_file.write('rm -rf %{buildroot}/*\n') + for package_name in spec.packages.keys(): + spec_file.write(f"cp -af %{{_builddir}}/%{{name}}-%{{version}}/debian/{package_name}/* %{{buildroot}}\n") + + spec_file.write("rm -rf %{buildroot}/DEBIAN\n") + + def copy2source(self): + """ + 将所有文件复制到source根目录下 + """ + all_files = [os.path.join(self.source_path, x) for x in os.listdir(self.source_path) + if not os.path.isdir(os.path.join(self.source_path, x))] + for file in all_files: + shutil.copy(file, self.rpm_source_path) + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("One target!") + sys.exit() + + target = sys.argv[1] + SpecGenarator(target) diff --git a/tools/deb2rpm/scripts/sqlite_database.py b/tools/deb2rpm/scripts/sqlite_database.py new file mode 100644 index 0000000000000000000000000000000000000000..4c2d1192517c9e47998874b6ab47a146a145f178 --- /dev/null +++ b/tools/deb2rpm/scripts/sqlite_database.py @@ -0,0 +1,210 @@ +import sqlite3 +import os +import re + +from typing import List + +home = os.getenv('HOME') +db_name = f"{home}/deb2rpm/database/deb2rpm.db" + +def get_line_info(line: str) -> List: + """ + 从line中提取信息 + @param line 一行，格式为key: value + @return 返回[key, value] + """ + pattern = r'^(.*?):\s*(.*)' + match = re.search(pattern, line) + if match: + return match.group(1).strip(), match.group(2).strip() + return "", "" + +class BaseDB: + """ + 用于存储源文件名和依赖的数据库 + """ + def __init__(self): + self.conn = sqlite3.connect(db_name) + # print(f"成功连接数据库{db_name}") + self.cursor = self.conn.cursor() + + def create_table(self, table_name: str, keys: List, types: List): + """ + 如果名为table_name的表不存在，则创建表 + @param table_name 表名 + @param keys 键名的列表 + @param types 各个键对应的类型 + """ + columns = ', '.join([f'{key} {key_type}' for key, key_type in zip(keys, types)]) + self.cursor.execute(f'CREATE TABLE IF NOT EXISTS {table_name} ({columns})') + self.conn.commit() + + def insert_data(self, table_name: str, data: List): + """ + 向table_name中插入data(类型均为字符串型) + @param table_name 表名 + @param data 数据列表 + """ + datas = ', '.join([f"'{value}'" for value in data]) + self.cursor.execute(f'INSERT INTO {table_name} VALUES ({datas})') + self.conn.commit() + + def select_data(self, table_name: str, key: str, value: str, columns=None) -> List: + """ + 查询表，条件为key = value + @param table_name 表名 + @param key 条件中的键名 + @param value 条件中的值 + @param columns 要查询的列(默认为全部) + @return 符合条件的行列表 + """ + if columns: + select_columns = ', '.join(columns) + else: + select_columns = '*' + self.cursor.execute(f"SELECT {select_columns} FROM {table_name} WHERE {key} = '{value}'") + rows = self.cursor.fetchone() + return rows + + def delete_data(self, table_name:str, key: str, value: str): + """ + 删除满足key = value的数据 + @param table_name 表名 + @param key 条件中的键名 + @param value 条件中的值 + """ + self.cursor.execute(f"DELETE FROM {table_name} WHERE {key} = '{value}'") + self.commit() + + def table_exists(self, table_name: str) -> bool: + """ + 判断数据库中是否存在名为table_name的表 + @param table_name 表名 + @return 存在则返回True，不存在返回False + """ + self.cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'") + result = self.cursor.fetchone() + + if result: + return True + return False + + def drop_table(self, table_name: str): + """ + 删除表table_name + """ + self.cursor.execute(f'DROP TABLE IF EXISTS {table_name}') + self.conn.commit() + + +class SourceDB(BaseDB): + def __init__(self, table_name: str, path: str): + """ + 初始化source表 + @param table_name source表名 + @param path source文件路径 + """ + super().__init__() + self.table_name = table_name + self.keys = ['Package', 'Version', 'BuildDepends', 'Directory', 'Packages', 'Files'] + self.types = ['TinyText', 'TinyText', 'Text', 'TinyText', 'Text', 'Text'] + + super().create_table(table_name, self.keys, self.types) + get_files = False + get_packages = False + package_name = "" + version = "" + build_depends = "" + directory = "" + packages = "" + files = "" + with open(path, 'r') as file: + for line in file: + key, value = get_line_info(line) + if (line.strip() == "" or key == "Checksums-Sha1") and get_files: + get_files = False + super().insert_data(table_name, [package_name, version, build_depends.strip(',').strip(), directory, packages, files.strip()]) + package_name = "" + version = "" + build_depends = "" + directory = "" + packages = "" + files = "" + continue + if key == 'Package': + package_name = value + elif key == 'Version': + version = value + elif key == 'Package-List': + get_packages = True + elif key == 'Build-Depends': + build_depends = value + elif key == 'Directory': + directory = value + elif key == 'Build-Depends-Indep' or key == 'Build-Depends-Arch': + build_depends += f', {value}' + elif key == 'Files': + get_files = True + get_packages = False + elif get_files: + part = line.strip().split() + files += f" {part[-1]}" + elif get_packages: + part = line.strip().split() + packages += f" {part[0]}" + + self.conn.close() + + def select_data(self, key: str, value: str, columns=None) -> List: + super().select_data(self.table_name, key, value, columns) + + def close_db(self): + self.conn.close() + +class PackageDB(BaseDB): + def __init__(self, table_name: str, path: str): + """ + 初始化package表 + @param table_name package表名 + @param path source文件路径 + """ + super().__init__() + self.table_name = table_name + self.keys = ['Package','Provides', 'PreDepends','Depends','Recommends', 'Suggests', 'Breaks', 'Replaces', 'Source', 'Filename'] + self.types = ['TinyText', 'TinyText','TinyText', 'Text', 'TinyText', 'TinyText', 'TinyText', 'TinyText', 'TinyText', 'TinyText'] + super().create_table(table_name, self.keys, self.types) + + data = {} + for key in self.keys: + data[key] = "" + with open(path, 'r') as file: + for line in file: + key, value = get_line_info(line) + if key == 'Size': + if data['Source'] == '': + data['Source'] = data['Package'] + super().insert_data(table_name, data.values()) + data = {} + for key in self.keys: + data[key] = "" + continue + if key in self.keys: + data[key] = value + if key == 'Pre-Depends': + data['PreDepends'] = value + + self.conn.close() + + def select_data(self, key: str, value: str, columns=None) -> List: + super().select_data(self.table_name, key, value, columns) + + def close_db(self): + self.conn.close() + +if __name__ == "__main__": + source_db = SourceDB('SOURCE', '/home/young/Source/source_test') + package_db = PackageDB('PACKAGE', '/home/young/Package/package-test') + db = BaseDB() + res = db.select_data('PACKAGE', 'Package', 'nginx') + print(res) + # print(res[0])