scripts/sanitize_proposals.py

#!/usr/bin/env python

import interactive
import sys


def sanitize_html(fragment):
  """Returns sanitized HTML fragment.

  Args:
    fragment: A string containing HTML fragment.

  Returns:
    A string containing sanitized HTML fragment.
  """
  from html5lib import html5parser
  from melange.utils import htmlsanitizer
  return ''.join(
      token.toxml() for token in
      html5parser.HTMLParser(tokenizer=htmlsanitizer.HTMLSanitizer)
          .parseFragment(fragment)
          .childNodes)

def sanitizeProposals():
  from summerofcode.models import proposal as proposal_model

  it = interactive.deepFetchNDB(lambda: proposal_model.Proposal.query())
  for proposal in it:
    file_name = proposal.key.urlsafe()
    if len(file_name) > 50:
      file_name = file_name[:50]

    oldFile = open('%s.old' % file_name, 'w')
    oldFile.write(proposal.content.encode('utf8'))
    oldFile.close()

    newFile = open('%s.new' % file_name, 'w')
    newFile.write(sanitize_html(proposal.content).encode('utf8'))
    newFile.close()


def sampleProposals(key_strings):
  from google.appengine.ext import ndb
  from summerofcode.models import proposal as proposal_model

  proposals = ndb.get_multi(
      ndb.Key(urlsafe=key_string) for key_string in key_strings)
  for proposal in proposals:
    file_name = proposal.key.urlsafe()
    if len(file_name) > 50:
      file_name = file_name[:50]

    oldFile = open('%s.old' % file_name, 'w')
    oldFile.write(proposal.content.encode('utf8'))
    oldFile.close()

    newFile = open('%s.new' % file_name, 'w')
    newFile.write(sanitize_html(proposal.content).encode('utf8'))
    newFile.close()

def main():
  interactive.setup()
  interactive.setDjango()

  context = {
      'sp': sanitizeProposals,
      'sample': sampleProposals
      }

  interactive.remote(sys.argv[1:], context)
  interactive.main(sys.argv[1:])

if __name__ == '__main__':
  if len(sys.argv) < 2:
    print "Usage: %s app_id [host]" % (sys.argv[0],)
    sys.exit(1)
  print ("Now you're at Python prompt, and you probably want to re-import "
      "this script and run functions from it")
  main()