edgedb 是基于python开发的,同时集成了cython 以下为包的setup.py 配置,从里面我们可以看到关于edgedb 的一些依赖
以及构建过程
setup.py 源码
整体配置不算很多,500 多行,主要是cython extension 配置以及pg 构建配置,以及pg extension 配置,其中添加了关于pg 以及
pg 扩展build 的自定义cmdclass
- 代码
# # This source file is part of the EdgeDB open source project. # # Copyright 2008-present MagicStack Inc. and the EdgeDB authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os.path import pathlib import platform import shutil import subprocess import textwrap from distutils import extension as distutils_extension from distutils.command import build as distutils_build from distutils.command import build_ext as distutils_build_ext import setuptools from setuptools.command import develop as setuptools_develop RUNTIME_DEPS = [ 'asyncpg~=0.18.2', 'click~=6.7', 'httptools>=0.0.13', 'immutables>=0.9', 'parsing~=1.6.1', 'prompt_toolkit>=2.0.0', 'psutil~=5.6.1', 'Pygments~=2.3.0', 'setproctitle~=1.1.10', 'setuptools_scm~=3.2.0', 'uvloop~=0.12.2', 'graphql-core~=2.1.0', 'promise~=2.2.0', ] CYTHON_DEPENDENCY = 'Cython==0.29.6' DOCS_DEPS = [ 'Sphinx~=2.0.0', 'lxml~=4.2.5', 'sphinxcontrib-asyncio~=0.2.0', ] BUILD_DEPS = [ CYTHON_DEPENDENCY, ] EXTRA_DEPS = { 'test': [ 'flake8~=3.7.5', 'pycodestyle~=2.5.0', 'coverage~=4.5.2', 'requests-xml~=0.2.3', 'lxml', 'requests-xml', ] + DOCS_DEPS, 'docs': DOCS_DEPS, } EXT_CFLAGS = ['-O2'] EXT_LDFLAGS = [] if platform.uname().system != 'Windows': EXT_CFLAGS.extend([ '-std=c99', '-fsigned-char', '-Wall', '-Wsign-compare', '-Wconversion' ]) def _compile_parsers(build_lib, inplace=False): import parsing import edb.edgeql.parser.grammar.single as edgeql_spec import edb.edgeql.parser.grammar.block as edgeql_spec2 import edb.edgeql.parser.grammar.sdldocument as schema_spec base_path = pathlib.Path(__file__).parent.resolve() for spec in (edgeql_spec, edgeql_spec2, schema_spec): spec_path = pathlib.Path(spec.__file__).parent subpath = pathlib.Path(str(spec_path)[len(str(base_path)) + 1:]) pickle_name = spec.__name__.rpartition('.')[2] + '.pickle' pickle_path = subpath / pickle_name cache = build_lib / pickle_path cache.parent.mkdir(parents=True, exist_ok=True) parsing.Spec(spec, pickleFile=str(cache), verbose=True) if inplace: shutil.copy2(cache, base_path / pickle_path) def _compile_build_meta(build_lib, version, pg_config, runstatedir): import pkg_resources from edb.server import buildmeta parsed_version = buildmeta.parse_version( pkg_resources.parse_version(version)) vertuple = list(parsed_version._asdict().values()) vertuple[2] = int(vertuple[2]) vertuple = tuple(vertuple) content = textwrap.dedent('''\ # # This source file is part of the EdgeDB open source project. # # Copyright 2008-present MagicStack Inc. and the EdgeDB authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # # THIS FILE HAS BEEN AUTOMATICALLY GENERATED. # PG_CONFIG_PATH = {pg_config!r} RUNSTATE_DIR = {runstatedir!r} VERSION = {version!r} ''').format(version=vertuple, pg_config=pg_config, runstatedir=runstatedir) directory = build_lib / 'edb' / 'server' if not directory.exists(): directory.mkdir(parents=True) with open(directory / '_buildmeta.py', 'w+t') as f: f.write(content) def _compile_postgres(build_base, *, force_build=False, fresh_build=True, run_configure=True, build_contrib=True): proc = subprocess.run( ['git', 'submodule', 'status', 'postgres'], stdout=subprocess.PIPE, universal_newlines=True, check=True) status = proc.stdout if status[0] == '-': print( 'postgres submodule not initialized, ' 'run `git submodule init; git submodule update`') exit(1) proc = subprocess.run( ['git', 'submodule', 'status', 'postgres'], stdout=subprocess.PIPE, universal_newlines=True, check=True) revision, _, _ = proc.stdout[1:].partition(' ') source_stamp = proc.stdout[0] + revision postgres_build = (build_base / 'postgres').resolve() postgres_src = (pathlib.Path(__file__).parent / 'postgres').resolve() postgres_build_stamp = postgres_build / 'stamp' if postgres_build_stamp.exists(): with open(postgres_build_stamp, 'r') as f: build_stamp = f.read() else: build_stamp = None is_outdated = source_stamp != build_stamp if is_outdated or force_build: system = platform.system() if system == 'Darwin': uuidlib = 'e2fs' elif system == 'Linux': uuidlib = 'e2fs' else: raise NotImplementedError('unsupported system: {}'.format(system)) if fresh_build and postgres_build.exists(): shutil.rmtree(postgres_build) build_dir = postgres_build / 'build' if not build_dir.exists(): build_dir.mkdir(parents=True) if run_configure or fresh_build or is_outdated: subprocess.run([ str(postgres_src / 'configure'), '--prefix=' + str(postgres_build / 'install'), '--with-uuid=' + uuidlib, ], check=True, cwd=str(build_dir)) subprocess.run( ['make', 'MAKELEVEL=0', '-j', str(max(os.cpu_count() - 1, 1))], cwd=str(build_dir), check=True) if build_contrib or fresh_build or is_outdated: subprocess.run( [ 'make', '-C', 'contrib', 'MAKELEVEL=0', '-j', str(max(os.cpu_count() - 1, 1)) ], cwd=str(build_dir), check=True) subprocess.run( ['make', 'MAKELEVEL=0', 'install'], cwd=str(build_dir), check=True) if build_contrib or fresh_build or is_outdated: subprocess.run( ['make', '-C', 'contrib', 'MAKELEVEL=0', 'install'], cwd=str(build_dir), check=True) with open(postgres_build_stamp, 'w') as f: f.write(source_stamp) def _compile_postgres_extensions(build_base): postgres_build = (build_base / 'postgres').resolve() postgres_build_stamp_path = postgres_build / 'stamp' ext_build = (build_base / 'ext').resolve() ext_build_stamp_path = ext_build / 'stamp' if postgres_build_stamp_path.exists(): with open(postgres_build_stamp_path, 'r') as f: postgres_build_stamp = f.read() else: raise RuntimeError('Postgres is not built, cannot build extensions') if ext_build_stamp_path.exists(): with open(ext_build_stamp_path, 'r') as f: ext_build_stamp = f.read() else: ext_build_stamp = None ext_dir = (pathlib.Path(__file__).parent / 'ext').resolve() pg_config = (build_base / 'postgres' / 'install' / 'bin' / 'pg_config').resolve() if not ext_dir.exists(): raise RuntimeError('missing Postgres extension directory') ext_make = ['make', '-C', str(ext_dir), 'PG_CONFIG=' + str(pg_config)] if ext_build_stamp != postgres_build_stamp: print('Extensions build stamp does not match Postgres build stamp. ' 'Rebuilding...') subprocess.run(ext_make + ['clean'], check=True) ext_build.mkdir(parents=True, exist_ok=True) subprocess.run(ext_make, check=True) subprocess.run(ext_make + ['install'], check=True) ext_build_stamp = postgres_build_stamp with open(ext_build_stamp_path, 'w') as f: f.write(ext_build_stamp) class build(distutils_build.build): user_options = distutils_build.build.user_options + [ ('pg-config=', None, 'path to pg_config to use with this build'), ('runstatedir=', None, 'directory to use for the runtime state'), ] def initialize_options(self): super().initialize_options() self.pg_config = None self.runstatedir = None def finalize_options(self): super().finalize_options() def run(self, *args, **kwargs): super().run(*args, **kwargs) build_lib = pathlib.Path(self.build_lib) _compile_parsers(build_lib) if self.pg_config: _compile_build_meta( build_lib, self.distribution.metadata.version, self.pg_config, self.runstatedir, ) class develop(setuptools_develop.develop): def run(self, *args, **kwargs): _compile_parsers(pathlib.Path('build/lib'), inplace=True) _compile_postgres(pathlib.Path('build').resolve()) _compile_postgres_extensions(pathlib.Path('build').resolve()) scripts = self.distribution.entry_points['console_scripts'] patched_scripts = [s + '_dev' for s in scripts if not s.startswith('edgedb-server')] patched_scripts.append('edb = edb.tools.edb:edbcommands') self.distribution.entry_points['console_scripts'] = patched_scripts super().run(*args, **kwargs) class build_postgres(setuptools.Command): description = "build postgres" user_options = [ ('configure', None, 'run ./configure'), ('build-contrib', None, 'build contrib'), ('fresh-build', None, 'rebuild from scratch'), ] def initialize_options(self): self.configure = False self.build_contrib = False self.fresh_build = False def finalize_options(self): pass def run(self, *args, **kwargs): _compile_postgres( pathlib.Path('build').resolve(), force_build=True, fresh_build=self.fresh_build, run_configure=self.configure, build_contrib=self.build_contrib) _compile_postgres_extensions( pathlib.Path('build').resolve()) class build_postgres_ext(setuptools.Command): description = "build postgres extensions" user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self, *args, **kwargs): _compile_postgres_extensions( pathlib.Path('build').resolve()) class build_ext(distutils_build_ext.build_ext): user_options = distutils_build_ext.build_ext.user_options + [ ('cython-annotate', None, 'Produce a colorized HTML version of the Cython source.'), ('cython-directives=', None, 'Cython compiler directives'), ] def initialize_options(self): # initialize_options() may be called multiple times on the # same command object, so make sure not to override previously # set options. if getattr(self, '_initialized', False): return super(build_ext, self).initialize_options() if os.environ.get('EDGEDB_DEBUG'): self.cython_always = True self.cython_annotate = True self.cython_directives = "linetrace=True" self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL' self.debug = True else: self.cython_always = False self.cython_annotate = None self.cython_directives = None self.debug = False def finalize_options(self): # finalize_options() may be called multiple times on the # same command object, so make sure not to override previously # set options. if getattr(self, '_initialized', False): return import pkg_resources # Double check Cython presence in case setup_requires # didn't go into effect (most likely because someone # imported Cython before setup_requires injected the # correct egg into sys.path. try: import Cython except ImportError: raise RuntimeError( 'please install {} to compile edgedb from source'.format( CYTHON_DEPENDENCY)) cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY) if Cython.__version__ not in cython_dep: raise RuntimeError( 'edgedb requires {}, got Cython=={}'.format( CYTHON_DEPENDENCY, Cython.__version__ )) from Cython.Build import cythonize directives = { 'language_level': '3' } if self.cython_directives: for directive in self.cython_directives.split(','): k, _, v = directive.partition('=') if v.lower() == 'false': v = False if v.lower() == 'true': v = True directives[k] = v self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules, compiler_directives=directives, annotate=self.cython_annotate, include_path=["edb/server/pgproto/"]) super(build_ext, self).finalize_options() setuptools.setup( setup_requires=RUNTIME_DEPS + BUILD_DEPS, use_scm_version=True, name='edgedb-server', description='EdgeDB Server', author='MagicStack Inc.', author_email='hello@magic.io', packages=['edb'], include_package_data=True, cmdclass={ 'build': build, 'build_ext': build_ext, 'develop': develop, 'build_postgres': build_postgres, 'build_postgres_ext': build_postgres_ext, }, entry_points={ 'console_scripts': [ 'edgedb = edb.cli:cli', 'edgedb-server = edb.server.main:main', ] }, ext_modules=[ distutils_extension.Extension( "edb.server.pgproto.pgproto", ["edb/server/pgproto/pgproto.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.dbview.dbview", ["edb/server/dbview/dbview.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.mng_port.edgecon", ["edb/server/mng_port/edgecon.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.cache.stmt_cache", ["edb/server/cache/stmt_cache.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.pgcon.pgcon", ["edb/server/pgcon/pgcon.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.http.http", ["edb/server/http/http.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.http_edgeql_port.protocol", ["edb/server/http_edgeql_port/protocol.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.http_graphql_port.protocol", ["edb/server/http_graphql_port/protocol.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), ], install_requires=RUNTIME_DEPS, extras_require=EXTRA_DEPS, test_suite='tests.suite', )
代码说明
核心代码还是setuptools.setup 中的内容,一种包含了依赖包以及扩展的配置
- 依赖
依赖包含了运行时依赖以及构建依赖
运行时依赖,主要是一些核心库,包括了pg 连接库 http 操作的,graphql 依赖库,cli解析的 RUNTIME_DEPS = [ 'asyncpg~=0.18.2', 'click~=6.7', 'httptools>=0.0.13', 'immutables>=0.9', 'parsing~=1.6.1', 'prompt_toolkit>=2.0.0', 'psutil~=5.6.1', 'Pygments~=2.3.0', 'setproctitle~=1.1.10', 'setuptools_scm~=3.2.0', 'uvloop~=0.12.2', 'graphql-core~=2.1.0', 'promise~=2.2.0', ] CYTHON_DEPENDENCY = 'Cython==0.29.6' BUILD_DEPS = [ CYTHON_DEPENDENCY, ]
- 包名称
edgedb 基本上所有python相关的代码都在edb 中
packages=['edb'],
- 自定义cmdclass 以及entry_points
详细的下边细说
# 添加了关于(pg 以及edgedb)构建,(cython,pg)扩展构建 cmdclass={ 'build': build, 'build_ext': build_ext, 'develop': develop, 'build_postgres': build_postgres, 'build_postgres_ext': build_postgres_ext, }, entry_points={ 'console_scripts': [ 'edgedb = edb.cli:cli', 'edgedb-server = edb.server.main:main', ] }
- 依赖的cython extension 配置
主要是cython 配置,暴露的包名以及cython 路径,以及编译选项
ext_modules=[ distutils_extension.Extension( "edb.server.pgproto.pgproto", ["edb/server/pgproto/pgproto.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.dbview.dbview", ["edb/server/dbview/dbview.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.mng_port.edgecon", ["edb/server/mng_port/edgecon.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.cache.stmt_cache", ["edb/server/cache/stmt_cache.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.pgcon.pgcon", ["edb/server/pgcon/pgcon.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.http.http", ["edb/server/http/http.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.http_edgeql_port.protocol", ["edb/server/http_edgeql_port/protocol.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), distutils_extension.Extension( "edb.server.http_graphql_port.protocol", ["edb/server/http_graphql_port/protocol.pyx"], extra_compile_args=EXT_CFLAGS, extra_link_args=EXT_LDFLAGS), ], install_requires=RUNTIME_DEPS, extras_require=EXTRA_DEPS,
cmdclass 说明
cmdclass 是edgedb 包的核心部分,包含了cython 构建以及pg 源码构建pg extension 扩展构建
- build_ext 说明
主要是cython 相关的配置
class build_ext(distutils_build_ext.build_ext): user_options = distutils_build_ext.build_ext.user_options + [ ('cython-annotate', None, 'Produce a colorized HTML version of the Cython source.'), ('cython-directives=', None, 'Cython compiler directives'), ] def initialize_options(self): # initialize_options() may be called multiple times on the # same command object, so make sure not to override previously # set options. if getattr(self, '_initialized', False): return super(build_ext, self).initialize_options() if os.environ.get('EDGEDB_DEBUG'): self.cython_always = True self.cython_annotate = True self.cython_directives = "linetrace=True" self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL' self.debug = True else: self.cython_always = False self.cython_annotate = None self.cython_directives = None self.debug = False def finalize_options(self): # finalize_options() may be called multiple times on the # same command object, so make sure not to override previously # set options. if getattr(self, '_initialized', False): return import pkg_resources # Double check Cython presence in case setup_requires # didn't go into effect (most likely because someone # imported Cython before setup_requires injected the # correct egg into sys.path. try: import Cython except ImportError: raise RuntimeError( 'please install {} to compile edgedb from source'.format( CYTHON_DEPENDENCY)) cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY) if Cython.__version__ not in cython_dep: raise RuntimeError( 'edgedb requires {}, got Cython=={}'.format( CYTHON_DEPENDENCY, Cython.__version__ )) from Cython.Build import cythonize directives = { 'language_level': '3' } if self.cython_directives: for directive in self.cython_directives.split(','): k, _, v = directive.partition('=') if v.lower() == 'false': v = False if v.lower() == 'true': v = True directives[k] = v self.distribution.ext_modules[:] = cythonize( self.distribution.ext_modules, compiler_directives=directives, annotate=self.cython_annotate, include_path=["edb/server/pgproto/"]) super(build_ext, self).finalize_options()
- build_postgres pg 构建配置
pg 是edgedb 的核心,build_postgres 提供了pg 源码编译的处理
class build_postgres(setuptools.Command): description = "build postgres" user_options = [ ('configure', None, 'run ./configure'), ('build-contrib', None, 'build contrib'), ('fresh-build', None, 'rebuild from scratch'), ] def initialize_options(self): self.configure = False self.build_contrib = False self.fresh_build = False def finalize_options(self): pass def run(self, *args, **kwargs): _compile_postgres( pathlib.Path('build').resolve(), force_build=True, fresh_build=self.fresh_build, run_configure=self.configure, build_contrib=self.build_contrib) _compile_postgres_extensions( pathlib.Path('build').resolve())
- build_postgres_ext 提供pg 扩展的构建处理
目前主要的扩展是提供一些工具类的封装扩展名称为 edbsys,代码在ext 中
class build_postgres_ext(setuptools.Command): description = "build postgres extensions" user_options = [] def initialize_options(self): pass def finalize_options(self): pass def run(self, *args, **kwargs): _compile_postgres_extensions( pathlib.Path('build').resolve())
- build 配置
主要提供了pg_config 的配置,指定后边pg 以及pg 扩展构建的环境
class build(distutils_build.build): user_options = distutils_build.build.user_options + [ ('pg-config=', None, 'path to pg_config to use with this build'), ('runstatedir=', None, 'directory to use for the runtime state'), ] def initialize_options(self): super().initialize_options() self.pg_config = None self.runstatedir = None def finalize_options(self): super().finalize_options() def run(self, *args, **kwargs): super().run(*args, **kwargs) build_lib = pathlib.Path(self.build_lib) _compile_parsers(build_lib) if self.pg_config: _compile_build_meta( build_lib, self.distribution.metadata.version, self.pg_config, self.runstatedir, )
说明
以上只是一个简单的分析,这个文件主要是为了进行edgedb 打包构建处理的,里面也使用了关于cython 的技术
还是值得学习的
参考资料
https://github.com/edgedb/edgedb
https://medium.com/@shamir.stav_83310/making-your-c-library-callable-from-python-by-wrapping-it-with-cython-b09db35012a3
https://github.com/stavshamir/cython-c-wrapper/
https://cython.readthedocs.io/en/latest/src/tutorial/external.html
https://cython.readthedocs.io/en/latest/src/tutorial/clibraries.html
http://pages.cs.wisc.edu/~yezheng/post/cython/
https://github.com/rongfengliang/cython-c-pip-demo/tree/local_source
来源:https://www.cnblogs.com/rongfengliang/p/10756106.html