| # -*- coding: utf-8 -*- |
| |
| """ |
| Example Usage |
| ============= |
| |
| The following commands can be run from the root directory of the Mercurial |
| repo. To run ``paver``, however, you'll need to do ``easy_install Paver``. |
| Most of the following commands accept other arguments; see ``command --help`` |
| for more information, or ``paver help`` for a list of all the valid commands. |
| |
| ``paver build`` |
| Builds the project. This essentially just runs a bunch of other tasks, |
| like ``pylint`` and ``tinymce_zip``, etc. |
| ``paver pylint`` |
| Runs PyLint on the project. |
| ``paver tinymce_zip`` |
| Builds the TinyMCE zip file. |
| |
| If you specify ``--dry-run`` before a task, then the action of that task will |
| not actually be carried out, although logging output will be displayed as if |
| it were. For example, you could run ``paver --dry-run tinymce_zip`` to see what |
| files would be added to the ``tinymce.zip`` file, etc. |
| """ |
| |
| import cStringIO |
| import os |
| import shutil |
| import sys |
| import zipfile |
| |
| from google.appengine.ext import testbed |
| |
| from epydoc import cli |
| from paver import easy |
| from paver import path |
| from paver import tasks |
| from pylint import lint |
| |
| # Paver comes with Jason Orendorff's 'path' module; this makes path |
| # manipulation easy and far more readable. |
| PROJECT_DIR = path.path(__file__).dirname().abspath() |
| REPORTS_DIR = PROJECT_DIR / 'reports' |
| JS_DIRS = ['soc/content/js'] |
| COPY_DIRS = ['soc/content/css'] |
| DONT_COPY_DIRS = [] # subset of COPY_DIRS to exclude |
| APP_FILES = [ |
| 'app.yaml', |
| 'index.yaml', |
| 'queue.yaml', |
| 'cron.yaml', |
| 'mapreduce.yaml', |
| 'main.py', |
| 'settings.py', |
| 'urls.py', |
| 'gae_django.py', |
| 'profiler.py', |
| 'appengine_config.py', |
| ] |
| APP_DIRS = [ |
| 'melange', |
| 'soc', |
| 'feedparser', |
| 'djangoforms', |
| 'ranklist', |
| 'shell', |
| 'html5lib', |
| 'gviz', |
| 'webmaster', |
| 'mapreduce', |
| 'summerofcode', |
| 'codein', |
| 'apiclient', |
| 'httplib2', |
| 'oauth2client', |
| 'uritemplate', |
| ] |
| CSS_FILES = { |
| 'jquery-ui/jquery.ui.merged.css': [ |
| 'jquery-ui/jquery.ui.core.css', |
| 'jquery-ui/jquery.ui.resizable.css', |
| 'jquery-ui/jquery.ui.selectable.css', |
| 'jquery-ui/jquery.ui.accordion.css', |
| 'jquery-ui/jquery.ui.autocomplete.css', |
| 'jquery-ui/jquery.ui.button.css', |
| 'jquery-ui/jquery.ui.dialog.css', |
| 'jquery-ui/jquery.ui.slider.css', |
| 'jquery-ui/jquery.ui.tabs.css', |
| 'jquery-ui/jquery.ui.datepicker.css', |
| 'jquery-ui/jquery.ui.progressbar.css', |
| 'jquery-ui/jquery.ui.theme.css', |
| ], |
| } |
| CSS_DIRS = ['soc/content/css/gsoc/', 'soc/content/css/gci'] |
| ZIP_FILES = ['tiny_mce.zip'] |
| DOCS_CONFIG = PROJECT_DIR / 'docs.config' |
| DOCS_OUTPUT = REPORTS_DIR / 'docs' |
| |
| # TODO(nathaniel): Get rid of all the "overrides" stuff as part of |
| # finishing https://code.google.com/p/soc/issues/detail?id=1560. |
| OVERRIDES_FOLDER = PROJECT_DIR / 'overrides' |
| OVERRIDES_DIRS = ['soc', 'soc/models', 'soc/content'] |
| OVERRIDES_FILES = ['soc/models/universities.py'] |
| |
| BUILD_BUNCH = easy.Bunch( |
| project_dir=PROJECT_DIR, |
| app_files=APP_FILES, |
| app_dirs=APP_DIRS, |
| app_build=PROJECT_DIR / 'build', |
| app_folder=PROJECT_DIR / 'app', |
| css_dirs=CSS_DIRS, |
| css_files=CSS_FILES, |
| zip_files=ZIP_FILES, |
| docs_config=DOCS_CONFIG, |
| docs_output=DOCS_OUTPUT, |
| copy_dirs=COPY_DIRS, |
| dont_copy_dirs=DONT_COPY_DIRS, |
| overrides_folder=OVERRIDES_FOLDER, |
| overrides_dirs=OVERRIDES_DIRS, |
| overrides_files=OVERRIDES_FILES, |
| skip_docs=False, |
| skip_pylint=False) |
| |
| PYLINT_APP_FOLDER_MODULES = [ |
| 'codein', |
| 'melange', |
| 'soc', |
| 'summerofcode', |
| 'settings.py', |
| 'urls.py', |
| 'main.py', |
| ] |
| PYLINT_TESTS_FOLDER = PROJECT_DIR / 'tests' |
| PYLINT_PROJECT_FOLDER_MODULES = ['pavement.py', 'setup.py'] |
| PYLINT_VERBOSE_ARGS = [ |
| # In the rcfile(pylintrc) errors-only option is set. This is to enable |
| # other messages as well. |
| # R and C modules are just too chatty, we can however turn a few of the |
| # more useful ones on explicitly. |
| '--enable=W,F', |
| '--reports=yes', |
| # We may want to enable these in the future |
| '--disable=protected-access,attribute-defined-outside-init', |
| # TODO(nathaniel): fix all occurences and enable this |
| '--disable=abstract-method', |
| # These are just plain useless, we don't ever want to these |
| '--disable=fixme,unused-argument,star-args,bad-builtin,locally-disabled', |
| # These are somewhat debatable, but not realistic for Melange |
| '--disable=no-init,super-init-not-called', |
| # TODO(nathaniel): fix all occurences and move this to pylintrc file. |
| '--enable=line-too-long', |
| ] |
| PYLINT_BUNCH = easy.Bunch( |
| app_folder_modules=PYLINT_APP_FOLDER_MODULES, |
| project_folder_modules=PYLINT_PROJECT_FOLDER_MODULES, |
| tests_folder=PYLINT_TESTS_FOLDER, |
| verbose=False, |
| verbose_args=PYLINT_VERBOSE_ARGS, |
| pylint_args=[], |
| with_module=None, |
| ignore=False, |
| **BUILD_BUNCH) |
| |
| # Install the option bunches. |
| easy.options( |
| build=BUILD_BUNCH, |
| clean_build=BUILD_BUNCH, |
| tinymce_zip=BUILD_BUNCH, |
| pylint=PYLINT_BUNCH) |
| |
| |
| # Utility functions |
| |
| def tinymce_zip_files(tiny_mce_dir): |
| """Yields each filename which should go into ``tiny_mce.zip``.""" |
| for filename in tiny_mce_dir.walkfiles(): |
| if '.svn' in filename.splitall(): |
| continue |
| |
| tasks.environment.info('%-4stiny_mce.zip <- %s', '', filename) |
| arcname = tiny_mce_dir.relpathto(filename) |
| yield filename, arcname |
| |
| |
| def write_zip_file(zip_file_handle, files): |
| if tasks.environment.dry_run: |
| for args in files: |
| pass |
| return |
| zip_file = zipfile.ZipFile(zip_file_handle, mode='w') |
| for args in files: |
| zip_file.write(*args) |
| zip_file.close() |
| |
| |
| def symlink(target, link_name): |
| if hasattr(target, 'symlink'): |
| target.symlink(link_name) |
| else: |
| # If we are on a platform where symlinks are not supported (such as |
| # Windows), simply copy the files across. |
| target.copy(link_name) |
| |
| # Tasks |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ('pylint-command=', 'c', 'Specify a custom pylint executable'), |
| ('with-module=', 'w', 'Include a specific module'), |
| ('verbose', 'v', 'Enables a lot of the noisy pylint output'), |
| ('ignore', 'i', 'Ignore PyLint errors') |
| ]) |
| def pylint(options): |
| """Check the source code using PyLint.""" |
| # Initial command. |
| arguments = [] |
| |
| if options.verbose: |
| arguments.extend(options.verbose_args) |
| |
| if 'pylint_args' in options: |
| arguments.extend(list(options.pylint_args)) |
| |
| # Add the list of paths containing the modules to check using PyLint. |
| if options.with_module: |
| arguments.append(options.with_module) |
| else: |
| arguments.extend(str(options.app_folder / module) |
| for module in options.app_folder_modules) |
| arguments.extend(str(options.project_dir / module) |
| for module in options.project_folder_modules) |
| # We should lint everything in the tests folder. |
| arguments.append(str(options.tests_folder)) |
| |
| # By placing run_pylint into its own function, it allows us to do dry runs |
| # without actually running PyLint. |
| def run_pylint(): |
| # Add app folder to path. |
| sys.path.insert(0, options.app_folder.abspath()) |
| # Add google_appengine directory to path. |
| gae_path = options.project_dir.abspath() / 'thirdparty' / 'google_appengine' |
| sys.path.insert(0, gae_path) |
| |
| # Specify PyLint RC file. |
| pylint_path = options.project_dir.abspath() / 'pylintrc' |
| arguments.append('--rcfile=' + pylint_path) |
| |
| # `lint.Run.__init__` runs the PyLint command. |
| try: |
| lint.Run(arguments) |
| # PyLint will `sys.exit()` when it has finished, so we need to catch |
| # the exception and process it accordingly. |
| except SystemExit, exc: |
| return_code = exc.args[0] |
| if return_code != 0 and (not options.pylint.ignore): |
| raise tasks.BuildFailure( |
| 'PyLint finished with a non-zero exit code: %d' % return_code) |
| |
| return easy.dry('pylint ' + ' '.join(arguments), run_pylint) |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('app-build=', 'b', 'App build directory (default /build)'), |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ('skip-pylint', 's', 'Skip PyLint checker'), |
| ('skip-docs', '', 'Skip documentation creation'), |
| ('ignore-pylint', 'i', 'Ignore results of PyLint (but run it anyway)'), |
| ('verbose-pylint', 'v', 'Make PyLint run verbosely'), |
| ]) |
| def build(options): |
| """Build the project.""" |
| # If `--skip-pylint` is not provided, run PyLint. |
| if not options.skip_pylint: |
| # If `--ignore-pylint` is provided, act as if `paver pylint --ignore` |
| # was run. Likewise for `--verbose-pylint`. |
| if options.get('ignore_pylint', False): |
| options.pylint.ignore = True |
| if options.get('verbose_pylint', False): |
| options.pylint.verbose = True |
| pylint(options) |
| |
| # Compile the css files into one |
| build_css(options) |
| |
| # Clean old generated zip files from the app folder. |
| clean_zip(options) |
| |
| # Clean the App build directory by removing and re-creating it. |
| clean_build(options) |
| |
| # Build the tiny_mce.zip file. |
| tinymce_zip(options) |
| |
| # Make the necessary symlinks between the app and build directories. |
| build_symlinks(options) |
| |
| # Handle overrides |
| overrides(options) |
| |
| # Handle deep overrides (copy) |
| deep_overrides(options) |
| |
| # Run grunt for production |
| run_grunt(options) |
| |
| # Builds documentation for the project |
| if not options.skip_docs: |
| build_docs(options) |
| |
| |
| @easy.task |
| def run_grunt(options): |
| """Run Grunt for build""" |
| easy.sh('bin/grunt build') |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('app-build=', 'b', 'App build directory (default /build)'), |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ]) |
| def build_symlinks(options): |
| """Build symlinks between the app and build folders.""" |
| # Create the symbolic links from the app folder to the build folder. |
| for filename in options.app_files + options.app_dirs + options.zip_files: |
| # The `symlink()` function handles discrepancies between platforms. |
| target = path.path(options.app_folder) / filename |
| link = path.path(options.app_build) / filename |
| easy.dry('%-4s%-20s <- %s' % ('', target, link), |
| lambda: symlink(target, link.abspath())) |
| |
| |
| @easy.task |
| def build_css(options): |
| """Compiles the css files into one.""" |
| for css_dir in options.css_dirs: |
| for target, components in options.css_files.iteritems(): |
| target = options.app_folder / css_dir / target |
| with target.open('w') as target_file: |
| for component in components: |
| source = options.app_folder / css_dir / component |
| easy.dry( |
| "cat %s >> %s" % (source, target), |
| lambda: shutil.copyfileobj(source.open('r'), target_file)) |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('app-build=', 'b', 'App build directory (default /build)'), |
| ]) |
| def clean_build(options): |
| """Clean the build folder.""" |
| # Not checking this could cause an error when trying to remove a |
| # non-existent file. |
| if path.path(options.app_build).exists(): |
| path.path(options.app_build).rmtree() |
| path.path(options.app_build).makedirs_p() |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ]) |
| def clean_zip(options): |
| """Remove all the generated zip files from the app folder.""" |
| for zip_file in options.zip_files: |
| zip_path = path.path(options.app_folder) / zip_file |
| if zip_path.exists(): |
| zip_path.remove() |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ]) |
| def tinymce_zip(options): |
| """Create the zip file containing TinyMCE. |
| |
| This zip file then gets served with google.appengine.ext.zipserve. |
| (Configured in app.yaml). |
| |
| Why? A long time ago, in a galaxy far away, App Engine had a limit |
| on the number of files that could be in an app, and Melange was |
| bumping up against that limit. |
| |
| This isn't strictly necessary anymore, but it works and there's no |
| reason to remove it. It probably makes the release process slightly |
| faster, because fewer files need to be uploaded. |
| |
| """ |
| tinymce_dir = path.path( |
| options.app_folder) / 'soc/content/js/thirdparty/tiny_mce' |
| tinymce_zip_filename = path.path(options.app_folder) / 'tiny_mce.zip' |
| if tasks.environment.dry_run: |
| tinymce_zip_fp = cStringIO.StringIO() |
| else: |
| # Ensure the parent directories exist. |
| tinymce_zip_filename.dirname().makedirs_p() |
| tinymce_zip_fp = open(tinymce_zip_filename, mode='w') |
| |
| try: |
| write_zip_file(tinymce_zip_fp, tinymce_zip_files(tinymce_dir)) |
| except Exception, exc: |
| tinymce_zip_fp.close() |
| tinymce_zip_filename.remove() |
| raise tasks.BuildFailure( |
| 'Error occurred creating tinymce.zip: %r' % (exc,)) |
| finally: |
| if not tinymce_zip_fp.closed: |
| tinymce_zip_fp.close() |
| |
| |
| @easy.task |
| @easy.cmdopts([ |
| ('docs-output=', '', 'Output directory for documentation'), |
| ('docs-config=', '', 'Configuration file for documentation'), |
| ]) |
| def build_docs(options): |
| """Builds documentation for the project.""" |
| # Epydoc smartly makes its own output directory if the output |
| # directory doesn't exist, but is not so smart that it will |
| # recursively create the parents of the output directory if they |
| # don't exist. Since we have the output directory path here we |
| # might as well create it rather than parsing out just the parents |
| # and leaving the directory itself to epydoc. |
| if not os.path.exists(options.docs_output): |
| os.makedirs(options.docs_output) |
| |
| # NOTE(nathaniel): Epydoc actually imports modules during analysis, |
| # Melange's modules in turn import App Engine modules, and App Engine |
| # modules complain if the right Django and App Engine settings aren't |
| # in place at import time. Consequently, we must mutate the current |
| # environment to be that of an App Engine test before we can build |
| # Melange's documentation. |
| os.environ['DJANGO_SETTINGS_MODULE'] = 'settings' |
| os.environ['SERVER_SOFTWARE'] = 'build' |
| appengine_testbed = testbed.Testbed() |
| appengine_testbed.activate() |
| appengine_testbed.init_datastore_v3_stub() |
| |
| # NOTE(nathaniel): Deriving the options to pass to (epydoc.)cli.main this |
| # way is horsehockey, but epydoc doesn't actually expose a proper API. |
| stored_actual_argv = sys.argv |
| sys.argv = [ |
| 'unused_fake_executuable', |
| '--config=%s' % options.docs_config, |
| '--output=%s' % options.docs_output, |
| ] |
| epydoc_options, epydoc_names = cli.parse_arguments() |
| sys.argv = stored_actual_argv |
| |
| # NOTE(nathaniel): As of 13 January 2014 this call emits two false |
| # positive "Bad argument - expected name or tuple" errors. See |
| # https://sourceforge.net/p/epydoc/bugs/363/ for progress. |
| cli.main(epydoc_options, epydoc_names) |
| |
| |
| @easy.task |
| def deep_overrides(options): |
| """Copies files from the copy structure to the build directory.""" |
| dirs = [options.app_folder / i for i in options.copy_dirs] |
| dont_copy_dirs = [options.app_folder / i for i in options.dont_copy_dirs] |
| |
| for source_dir in dirs: |
| dest_dir = options.app_build / options.app_folder.relpathto(source_dir) |
| dest_dir.remove() |
| source_dir.copytree(dest_dir) |
| |
| for remove_dir in dont_copy_dirs: |
| dest_dir = options.app_build / options.app_folder.relpathto(remove_dir) |
| dest_dir.rmtree() |
| |
| |
| @easy.task |
| def overrides(options): |
| """Copies files from the overrides structure to the build directory.""" |
| for override_path in options.overrides_dirs: |
| target = options.app_build / override_path |
| unroll_symlink(target) |
| for override_path in options.overrides_files: |
| target = options.overrides_folder / override_path |
| if not target.exists(): |
| continue |
| if not target.isfile(): |
| tasks.environment.info('target "%s" is not a file', target) |
| continue |
| to = options.app_build / override_path |
| to.remove() |
| target.symlink(to) |
| |
| |
| def unroll_symlink(target): |
| """Unrolls a symlink. |
| |
| Does the following if target is a directory symlink: |
| - removes the symlink |
| - creates a directory with the same name |
| - populates it with symlinks to individual files |
| |
| Otherwise does nothing. |
| """ |
| if not target.exists(): |
| tasks.environment.info('target "%s" does not exist', target) |
| return |
| if not target.isdir(): |
| tasks.environment.info('target "%s" is not a directory', target) |
| return |
| if not target.islink(): |
| tasks.environment.info('target "%s" is not a symlink', target) |
| return |
| |
| deref = target.readlinkabs() |
| target.remove() |
| target.mkdir() |
| |
| contents = deref.listdir() |
| |
| for symlink_path in contents: |
| symlink_path.symlink(target / symlink_path.name) |