| # -*- coding: utf-8 -*- |
| |
| """ |
| Example Usage |
| ============= |
| |
| The following commands can be run from the root directory of the Mercurial |
| repo. To run ``paver``, however, you'll need to do ``easy_install Paver``. |
| Most of the following commands accept other arguments; see ``command --help`` |
| for more information, or ``paver help`` for a list of all the valid commands. |
| |
| ``paver build`` |
| Builds the project. This essentially just runs a bunch of other tasks, |
| like ``pylint`` and ``tinymce_zip``, etc. |
| ``paver pylint`` |
| Runs PyLint on the project. |
| ``paver tinymce_zip`` |
| Builds the TinyMCE zip file. |
| |
| If you specify ``--dry-run`` before a task, then the action of that task will |
| not actually be carried out, although logging output will be displayed as if |
| it were. For example, you could run ``paver --dry-run tinymce_zip`` to see what |
| files would be added to the ``tinymce.zip`` file, etc. |
| """ |
| |
| from cStringIO import StringIO |
| import shutil |
| import sys |
| import zipfile |
| |
| import paver |
| import paver.easy |
| import paver.tasks |
| from paver.easy import * |
| from paver.path import path |
| |
| |
| # Paver comes with Jason Orendorff's 'path' module; this makes path |
| # manipulation easy and far more readable. |
| PROJECT_DIR = path(__file__).dirname().abspath() |
| |
| |
| # Set some default options. Having the options at the top of the file cleans |
| # the whole thing up and makes the behaviour a lot more configurable. |
| options( |
| build = Bunch( |
| project_dir = PROJECT_DIR, |
| app_build = PROJECT_DIR / 'build', |
| app_folder = PROJECT_DIR / 'app', |
| overrides_folder = PROJECT_DIR / 'overrides', |
| overrides_dirs = ['soc', 'soc/models'], |
| overrides_files = ['soc/models/universities.py'], |
| app_files = ['app.yaml', 'index.yaml', 'queue.yaml', 'cron.yaml', |
| 'mapreduce.yaml', 'main.py', 'settings.py', 'urls.py', |
| 'gae_django.py', 'profiler.py', 'appengine_config.py'], |
| app_dirs = ["soc", "feedparser", "djangoforms", |
| "jquery.min", "ranklist", "shell", "json.min", "jlinq", |
| "modernizr.min", "html5lib", "LABjs.min", "gviz","webmaster", |
| "gdata", "atom", "mapreduce"], |
| css_dirs = ["soc/content/css/v2/gsoc/", "soc/content/css/v2/gci"], |
| css_files = { |
| "jquery-ui/jquery.ui.merged.css": [ |
| "jquery-ui/jquery.ui.core.css", |
| "jquery-ui/jquery.ui.resizable.css", |
| "jquery-ui/jquery.ui.selectable.css", |
| "jquery-ui/jquery.ui.accordion.css", |
| "jquery-ui/jquery.ui.autocomplete.css", |
| "jquery-ui/jquery.ui.button.css", |
| "jquery-ui/jquery.ui.dialog.css", |
| "jquery-ui/jquery.ui.slider.css", |
| "jquery-ui/jquery.ui.tabs.css", |
| "jquery-ui/jquery.ui.datepicker.css", |
| "jquery-ui/jquery.ui.progressbar.css", |
| "jquery-ui/jquery.ui.theme.css" |
| ], |
| }, |
| zip_files = ['tiny_mce.zip'], |
| skip_pylint = False, |
| skip_closure = False, |
| ) |
| ) |
| |
| # The second call to options allows us to re-use some of the constants defined |
| # in the first call. |
| options( |
| clean_build = options.build, |
| tinymce_zip = options.build, |
| |
| pylint = Bunch( |
| check_modules = [ |
| 'soc/models', |
| 'soc/modules/gsoc/models', |
| 'soc/modules/gci/models', |
| #'soc/logic', |
| #'soc/modules/gsoc/logic', |
| #'soc/modules/gci/logic', |
| #'soc/views/helper', |
| #'soc/modules/gsoc/views/helper', |
| #'soc/modules/gci/views/helper', |
| 'reflistprop', |
| 'settings.py', |
| 'urls.py', |
| 'main.py', |
| ], |
| quiet = False, |
| quiet_args = [ |
| '--reports=no', |
| '--errors-only', |
| '--disable=E1103', |
| ], |
| pylint_args = [], |
| with_module = None, |
| ignore = False, |
| **options.build |
| ), |
| |
| closure = Bunch( |
| js_filter = None, |
| js_dir = None, |
| js_dirs = ["soc/content/js", "jquery", "jlinq", |
| "modernizr", "json", "LABjs"], |
| closure_bin = PROJECT_DIR / "thirdparty/closure/compiler.jar", |
| no_optimize = ["jquery-spin-1.1.1.js", "jquery-jqgrid.base.js"], |
| **options.build |
| ) |
| ) |
| |
| |
| # Utility functions |
| |
| def tinymce_zip_files(tiny_mce_dir): |
| """Yields each filename which should go into ``tiny_mce.zip``.""" |
| for filename in tiny_mce_dir.walkfiles(): |
| if '.svn' in filename.splitall(): |
| continue |
| |
| paver.tasks.environment.info( |
| '%-4stiny_mce.zip <- %s', '', filename) |
| arcname = tiny_mce_dir.relpathto(filename) |
| yield filename, arcname |
| |
| |
| def write_zip_file(zip_file_handle, files): |
| if paver.tasks.environment.dry_run: |
| for args in files: |
| pass |
| return |
| zip_file = zipfile.ZipFile(zip_file_handle, mode='w') |
| for args in files: |
| zip_file.write(*args) |
| zip_file.close() |
| |
| |
| def symlink(target, link_name): |
| if hasattr(target, 'symlink'): |
| target.symlink(link_name) |
| else: |
| # If we are on a platform where symlinks are not supported (such as |
| # Windows), simply copy the files across. |
| target.copy(link_name) |
| |
| |
| # Tasks |
| |
| |
| @task |
| @cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ('pylint-command=', 'c', 'Specify a custom pylint executable'), |
| ('with-module=', 'w', 'Include a specific module'), |
| ('quiet', 'q', 'Disables a lot of the pylint output'), |
| ('ignore', 'i', 'Ignore PyLint errors') |
| ]) |
| def pylint(options): |
| """Check the source code using PyLint.""" |
| from pylint import lint |
| |
| # Initial command. |
| arguments = [] |
| |
| if options.quiet: |
| arguments.extend(options.quiet_args) |
| if 'pylint_args' in options: |
| arguments.extend(list(options.pylint_args)) |
| |
| # Add the list of paths containing the modules to check using PyLint. |
| if options.with_module: |
| arguments.append(options.with_module) |
| else: |
| arguments.extend( |
| str(options.app_folder / module) for module in options.check_modules) |
| |
| |
| # By placing run_pylint into its own function, it allows us to do dry runs |
| # without actually running PyLint. |
| def run_pylint(): |
| # Add app folder to path. |
| sys.path.insert(0, options.app_folder.abspath()) |
| # Add google_appengine directory to path. |
| path = options.project_dir.abspath() / 'thirdparty' / 'google_appengine' |
| sys.path.insert(0, path) |
| |
| # Specify PyLint RC file. |
| path = options.project_dir.abspath() / 'scripts' / 'pylint' / 'pylintrc' |
| arguments.append('--rcfile=' + path) |
| |
| # `lint.Run.__init__` runs the PyLint command. |
| try: |
| lint.Run(arguments) |
| # PyLint will `sys.exit()` when it has finished, so we need to catch |
| # the exception and process it accordingly. |
| except SystemExit, exc: |
| return_code = exc.args[0] |
| if return_code != 0 and (not options.pylint.ignore): |
| raise paver.tasks.BuildFailure( |
| 'PyLint finished with a non-zero exit code') |
| |
| return dry('pylint ' + ' '.join(arguments), run_pylint) |
| |
| |
| @task |
| @cmdopts([ |
| ('app-build=', 'b', 'App build directory (default /build)'), |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ('skip-pylint', 's', 'Skip PyLint checker'), |
| ('ignore-pylint', 'i', 'Ignore results of PyLint (but run it anyway)'), |
| ('quiet-pylint', 'q', 'Make PyLint run quietly'), |
| ]) |
| def build(options): |
| """Build the project.""" |
| # If `--skip-pylint` is not provided, run PyLint. |
| if not options.skip_pylint: |
| # If `--ignore-pylint` is provided, act as if `paver pylint --ignore` |
| # was run. Likewise for `--quiet-pylint`. |
| if options.get('ignore_pylint', False): |
| options.pylint.ignore = True |
| if options.get('quiet_pylint', False): |
| options.pylint.quiet = True |
| pylint(options) |
| |
| # Compile the css files into one |
| build_css(options) |
| |
| # Clean old generated zip files from the app folder. |
| clean_zip(options) |
| |
| # Clean the App build directory by removing and re-creating it. |
| clean_build(options) |
| |
| # Build the tiny_mce.zip file. |
| tinymce_zip(options) |
| |
| # Make the necessary symlinks between the app and build directories. |
| build_symlinks(options) |
| |
| # Handle overrides |
| overrides(options) |
| |
| |
| @task |
| @cmdopts([ |
| ('app-build=', 'b', 'App build directory (default /build)'), |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ]) |
| def build_symlinks(options): |
| """Build symlinks between the app and build folders.""" |
| # Create the symbolic links from the app folder to the build folder. |
| for filename in options.app_files + options.app_dirs + options.zip_files: |
| # The `symlink()` function handles discrepancies between platforms. |
| target = path(options.app_folder) / filename |
| link = path(options.app_build) / filename |
| dry('%-4s%-20s <- %s' % ('', target, link), |
| lambda: symlink(target, link.abspath())) |
| |
| |
| @task |
| def build_css(options): |
| """Compiles the css files into one.""" |
| |
| for css_dir in options.css_dirs: |
| for target, components in options.css_files.iteritems(): |
| target = options.app_folder / css_dir / target |
| f = target.open('w') |
| |
| for component in components: |
| source = options.app_folder / css_dir / component |
| dry("cat %s >> %s" % (source, target), |
| lambda: shutil.copyfileobj(source.open('r'), f)) |
| f.close() |
| |
| |
| @task |
| @cmdopts([ |
| ('app-build=', 'b', 'App build directory (default /build)'), |
| ]) |
| def clean_build(options): |
| """Clean the build folder.""" |
| # Not checking this could cause an error when trying to remove a |
| # non-existent file. |
| if path(options.app_build).exists(): |
| path(options.app_build).rmtree() |
| path(options.app_build).makedirs_p() |
| |
| |
| @task |
| @cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ]) |
| def clean_zip(options): |
| """Remove all the generated zip files from the app folder.""" |
| for zip_file in options.zip_files: |
| zip_path = path(options.app_folder) / zip_file |
| if zip_path.exists(): |
| zip_path.remove() |
| |
| |
| @task |
| @cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ]) |
| def tinymce_zip(options): |
| """Create the zip file containing TinyMCE.""" |
| tinymce_dir = path(options.app_folder) / 'tiny_mce' |
| tinymce_zip_filename = path(options.app_folder) / 'tiny_mce.zip' |
| if paver.tasks.environment.dry_run: |
| tinymce_zip_fp = StringIO() |
| else: |
| # Ensure the parent directories exist. |
| tinymce_zip_filename.dirname().makedirs_p() |
| tinymce_zip_fp = open(tinymce_zip_filename, mode='w') |
| |
| try: |
| write_zip_file(tinymce_zip_fp, tinymce_zip_files(tinymce_dir)) |
| except Exception, exc: |
| tinymce_zip_fp.close() |
| tinymce_zip_filename.remove() |
| raise paver.tasks.BuildFailure( |
| 'Error occurred creating tinymce.zip: %r' % (exc,)) |
| finally: |
| if not tinymce_zip_fp.closed: |
| tinymce_zip_fp.close() |
| |
| def run_closure(f): |
| """Runs the closure compiler over one JS file""" |
| |
| tmp = f + ".tmp.js" |
| f.move(tmp) |
| |
| try: |
| sh("java -jar '%s' --js='%s' > '%s'" % (options.closure_bin, tmp, f)) |
| except BuildFailure, e: |
| paver.tasks.environment.error( |
| "%s minimization failed, copying plain file", f) |
| tmp.copy(f) |
| |
| tmp.remove() |
| |
| @task |
| @cmdopts([ |
| ('app-folder=', 'a', 'App folder directory (default /app)'), |
| ('js-dir=', 'j', 'JS directory to minimize, relative to /app'), |
| ('js-filter=', 'f', 'Minimize files matching this regex, default "*.js"'), |
| ]) |
| def closure(options): |
| """Runs the closure compiler over the JS files.""" |
| |
| if options.js_dir: |
| dirs = [options.app_folder / options.js_dir] |
| else: |
| dirs = [options.app_folder / i for i in options.js_dirs] |
| old_size = 0 |
| new_size = 0 |
| |
| js_filter = options.js_filter if options.js_filter else "*.js" |
| |
| for js_dir in dirs: |
| min_dir = js_dir + ".min" |
| |
| if not options.js_filter: |
| min_dir.rmtree() |
| |
| js_dir.copytree(min_dir) |
| |
| for f in min_dir.walkfiles(js_filter): |
| if f.name in options.no_optimize: |
| paver.tasks.environment.info( |
| '%-4sCLOSURE: Skipping %s', '', f) |
| continue |
| |
| paver.tasks.environment.info( |
| '%-4sCLOSURE: Processing %s', '', f) |
| |
| old_size += f.size |
| |
| run_closure(f) |
| |
| new_size += f.size |
| |
| rate = new_size*100 / old_size |
| paver.tasks.environment.info( |
| "%-4sCLOSURE: Source file sizes: %s, Dest file sizes: %s, Rate: %s", |
| '', old_size, new_size, rate) |
| |
| |
| @task |
| def overrides(options): |
| """Copies files from the overrides structure to the build directory. |
| """ |
| for path in options.overrides_dirs: |
| target = options.app_build / path |
| unroll_symlink(target) |
| for path in options.overrides_files: |
| target = options.overrides_folder / path |
| if not target.exists(): |
| continue |
| if not target.isfile(): |
| paver.tasks.environment.info( |
| "target '%s' is not a file", target) |
| continue |
| to = options.app_build / path |
| to.remove() |
| target.symlink(to) |
| |
| |
| def unroll_symlink(target): |
| """Unrolls a symlink. |
| |
| Does the following if target is a directory symlink: |
| - removes the symlink |
| - creates a directory with the same name |
| - populates it with symlinks to individual files |
| |
| Otherwise does nothing. |
| """ |
| if not target.exists(): |
| paver.tasks.environment.info( |
| "target '%s' does not exist", target) |
| return |
| if not target.isdir(): |
| paver.tasks.environment.info( |
| "target '%s' is not a directory", target) |
| return |
| if not target.islink(): |
| paver.tasks.environment.info( |
| "target '%s' is not a symlink", target) |
| return |
| |
| deref = target.readlinkabs() |
| target.remove() |
| target.mkdir() |
| |
| contents = deref.listdir() |
| |
| for path in contents: |
| path.symlink(target / path.name) |