app/soc/mapreduce/ndb_profile.py

# Copyright 2014 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""MapReduce scripts that convert profile entities to the new Profile model."""

import logging

from google.appengine.ext import db
from google.appengine.ext import ndb

from django.core import validators
from mapreduce import operation

from melange.models import address as address_model
from melange.models import connection as connection_model
from melange.models import contact as contact_model
from melange.models import education as education_model
from melange.models import profile as profile_model
from melange.models import user as user_model

# This MapReduce requires these models to have been imported.
from soc.models.profile import Profile
from soc.modules.gci.models.bulk_create_data import GCIBulkCreateData
from soc.modules.gci.models.organization import GCIOrganization
from soc.modules.gci.models.score import GCIOrgScore
from soc.modules.gci.models.score import GCIScore
from soc.modules.gci.models.profile import GCIProfile
from soc.modules.gci.models.task import GCITask
from soc.modules.gsoc.models.code_sample import GSoCCodeSample
from soc.modules.gsoc.models.comment import GSoCComment
from soc.modules.gsoc.models.grading_project_survey_record import GSoCGradingProjectSurveyRecord
from soc.modules.gsoc.models.grading_record import GSoCGradingRecord
from soc.modules.gsoc.models.profile import GSoCProfile
from soc.modules.gsoc.models.project import GSoCProject
from soc.modules.gsoc.models.project_survey_record import GSoCProjectSurveyRecord
from soc.modules.gsoc.models.proposal import GSoCProposal
from soc.modules.gsoc.models.score import GSoCScore

from summerofcode.models import survey as survey_model


@ndb.transactional
def _createProfileTxn(new_profile):
  """Persists the specified profile in the datastore."""
  new_profile.put()


def _teeStyleToEnum(profile):
  """Returns enum value for T-Shirt style for the specified profile.

  Args:
    profile: Profile entity.

  Returns:
    Value of profile_model.TeeStyle type corresponding to the T-Shirt style
    or profile_model.TeeStyle.NO_TEE, no T-Shirt style is set.
  """
  if not profile.tshirt_style:
    return profile_model.TeeStyle.NO_TEE
  elif profile.tshirt_style == 'male':
    return profile_model.TeeStyle.MALE
  elif profile.tshirt_style == 'female':
    return profile_model.TeeStyle.FEMALE
  else:
    logging.warning(
        'Unknown T-Shirt style %s for profile %s.',
        profile.tshirt_style, profile.key().name())
    return profile_model.TeeStyle.NO_TEE


def _teeSizeToEnum(profile):
  """Returns enum value for T-Shirt style for the specified profile.

  Args:
    profile: Profile entity.

  Returns:
    Value of profile_model.TeeSize type corresponding to the T-Shirt size
    or profile_model.TeeSize.NO_TEE, no T-Shirt size is set.
  """
  if not profile.tshirt_size:
    return profile_model.TeeSize.NO_TEE
  elif profile.tshirt_size == 'XXS':
    return profile_model.TeeSize.XXS
  elif profile.tshirt_size == 'XS':
    return profile_model.TeeSize.XS
  elif profile.tshirt_size == 'S':
    return profile_model.TeeSize.S
  elif profile.tshirt_size == 'M':
    return profile_model.TeeSize.M
  elif profile.tshirt_size == 'L':
    return profile_model.TeeSize.L
  elif profile.tshirt_size == 'XL':
    return profile_model.TeeSize.XL
  elif profile.tshirt_size == 'XXL':
    return profile_model.TeeSize.XXL
  elif profile.tshirt_size == 'XXXL':
    return profile_model.TeeSize.XXXL
  else:
    logging.warning(
        'Unknown T-Shirt size %s for profile %s.',
        profile.tshirt_size, profile.key().name())
    return profile_model.TeeSize.NO_TEE


def _genderToEnum(profile):
  """Returns enum value for gender for the specified profile.

  Args:
    profile: Profile entity.

  Returns:
    Value of profile_model.Gender type corresponding to the gender
    or profile_model.Gender.NOT_DISCLOSED, no gender is not ser.
  """
  if not profile.gender:
    return profile_model.Gender.NOT_DISCLOSED
  elif profile.gender == 'male':
    return profile_model.Gender.MALE
  elif profile.gender == 'female':
    return profile_model.Gender.FEMALE
  elif profile.gender == 'other':
    return profile_model.Gender.OTHER
  else:
    logging.warning(
        'Unknown gender %s for profile %s.',
        profile.gender, profile.key().name())
    return profile_model.Gender.NOT_DISCLOSED


def _statusToEnum(profile):
  """Returns enum value for status for the specified profile.

  Args:
    profile: Profile entity.

  Returns:
    Value of profile_model.Status type corresponding to the status.
  """
  if not profile.status or profile.status == 'active':
    return profile_model.Status.ACTIVE
  elif profile.status == 'invalid':
    return profile_model.Status.BANNED
  else:
    logging.warning(
        'Unknown status %s for profile %s.',
        profile.status, profile.key().name())
    return profile_model.Status.ACTIVE


def _degreeToEnum(profile):
  """Returns enum value for degree for the specified profile.

  Args:
    profile: Profile entity.

  Returns:
    Value of education_model.Degree type corresponding to the degree or None,
    if the degree is not recognized.
  """
  if not profile.student_info:
    raise ValueError('Profile is not a student.')
  elif profile.student_info.degree == 'Undergraduate':
    return education_model.Degree.UNDERGRADUATE
  elif profile.student_info.degree == 'Master':
    return education_model.Degree.MASTERS
  elif profile.student_info.degree == 'PhD':
    return education_model.Degree.PHD
  else:
    logging.warning(
        'Degree %s is not recognized for %s',
        profile.student_info.degree, profile.key().name())


def _getStudentData(profile):
  """Gets student data for the specified profile.

  Args:
    profile: Profile entity.

  Returns:
    Instance of profile_model.StudentData if the specified profile is a student
    or None otherwise.
  """
  if not profile.student_info:
    return None
  else:
    school_id = profile.student_info.school_name
    school_country = profile.student_info.school_country
    expected_graduation = profile.student_info.expected_graduation

    if isinstance(profile, GSoCProfile):
      properties = {
          'number_of_proposals': profile.student_info.number_of_proposals,
          'number_of_projects': profile.student_info.number_of_projects,
          'number_of_passed_evaluations':
              profile.student_info.passed_evaluations,
          'number_of_failed_evaluations':
              profile.student_info.failed_evaluations,
          'project_for_orgs': [ndb.Key.from_old_key(org_key) for org_key
              in profile.student_info.project_for_orgs]
          }

      if profile.student_info.tax_form:
        properties['tax_form'] = profile.student_info.getTaxFormKey()

      if profile.student_info.enrollment_form:
        properties['enrollment_form'] = (
            profile.student_info.getEnrollmentFormKey())

      degree = _degreeToEnum(profile)
      major = profile.student_info.major
      properties['education'] = education_model.Education(
          school_id=school_id, school_country=school_country,
          expected_graduation=expected_graduation, major=major, degree=degree)

      return profile_model.StudentData(**properties)
    else:
      properties = {
          'number_of_completed_tasks':
              profile.student_info.number_of_completed_tasks,
          }

      if profile.student_info.consent_form:
        properties['consent_form'] = (
            profile.student_info.consent_form.key())
        properties['is_consent_form_verified'] = (
            profile.student_info.consent_form_verified)

      if profile.student_info.student_id_form:
        properties['enrollment_form'] = (
            profile.student_info.student_id_form.key())
        properties['is_enrollment_form_verified'] = (
            profile.student_info.student_id_form_verified)

      if profile.student_info.winner_for:
        properties['winner_for'] = ndb.Key.from_old_key(
            profile.student_info.winner_for.key())

      grade = profile.student_info.grade
      properties['education'] = education_model.Education(
          school_id=school_id, school_country=school_country,
          expected_graduation=expected_graduation, grade=grade)

      return profile_model.StudentData(**properties)


def convertProfile(profile_key):
  """Converts the specified profile by creating a new user entity that inherits
  from the newly added NDB model.

  Args:
    profile: Profile key.
  """
  profile = db.get(profile_key)

  program = ndb.Key.from_old_key(
      Profile.program.get_value_for_datastore(profile))
  public_name = profile.public_name
  first_name = profile.given_name
  last_name = profile.surname

  if profile.photo_url and len(profile.photo_url) < 500:
    photo_url = profile.photo_url
  else:
    photo_url = None

  # create contact for profile
  try:
    email = validators.validate_email(profile.email)
  except Exception:
    logging.warning(
        'Invalid email %s for profile %s', profile.email, profile.key().name())
    email = None

  if profile.home_page and len(profile.home_page) < 500:
    web_page = profile.home_page
  else:
    web_page = None

  if profile.blog and len(profile.blog) < 500:
    blog = profile.blog
  else:
    blog = None

  phone = profile.phone
  contact = contact_model.Contact(
      email=email, web_page=web_page, blog=blog, phone=phone)

  # create residential address
  name = profile.full_name()
  street = profile.res_street
  street_extra = profile.res_street_extra
  city = profile.res_city
  province = profile.res_state
  country = profile.res_country
  postal_code = profile.res_postalcode
  residential_address = address_model.Address(
      name=name, street=street, street_extra=street_extra, city=city,
      province=province, country=country, postal_code=postal_code)

  # create shipping address
  if (profile.ship_street and profile.ship_city and profile.ship_country and
      profile.ship_postalcode):
    name = profile.ship_name or profile.full_name()
    street = profile.ship_street
    street_extra = profile.ship_street_extra
    city = profile.ship_city
    province = profile.ship_state
    country = profile.ship_country
    postal_code = profile.ship_postalcode
    shipping_address = address_model.Address(
        name=name, street=street, street_extra=street_extra, city=city,
        province=province, country=country, postal_code=postal_code)
  else:
    shipping_address = None

  birth_date = profile.birth_date
  tee_style = _teeStyleToEnum(profile)
  tee_size = _teeSizeToEnum(profile)
  gender = _genderToEnum(profile)
  program_knowledge = profile.program_knowledge

  student_data = _getStudentData(profile)
  mentor_for = set(
      ndb.Key.from_old_key(org_key) for org_key in profile.mentor_for)
  admin_for = set(
      ndb.Key.from_old_key(org_key) for org_key in profile.org_admin_for)

  status = _statusToEnum(profile)

  new_profile = profile_model.Profile(
      id=profile.key().name(),
      parent=ndb.Key.from_old_key(profile.parent_key()),
      program=program, public_name=public_name, first_name=first_name,
      last_name=last_name, photo_url=photo_url, contact=contact,
      residential_address=residential_address,
      shipping_address=shipping_address, birth_date=birth_date,
      tee_style=tee_style, tee_size=tee_size, gender=gender,
      program_knowledge=program_knowledge, student_data=student_data,
      mentor_for=mentor_for, admin_for=admin_for, status=status)

  _createProfileTxn(new_profile)


def _newKey(old_key):
  """Constructs new Profile key based on the specified GSoCProfile or
  GCIProfile key.

  Args:
    old_key: db.Key of GCIProfile or GSocProfile kind.

  Returns:
    db.Key of Profile kind.
  """
  return ndb.Key(
      user_model.User._get_kind(), old_key.parent().name(),
      profile_model.Profile._get_kind(), old_key.name()).to_old_key()


def _newProfileNDBKey(old_key):
  """Constructs new Profile key based on the specified GSoCProfile or
  GCIProfile key.

  Args:
    old_key: ndb.Key of GCIProfile or GSocProfile kind.

  Returns:
    ndb.Key of Profile kind.
  """
  return ndb.Key(
      user_model.User._get_kind(), old_key.parent().id(),
      profile_model.Profile._get_kind(), old_key.id())


def _convertReferenceProperty(model_property, entity):
  """Converts the specified ReferenceProperty whose value is either a key
  of GSoCProfile or GCIProfile type.

  Args:
    model_property: Property instance.
    entity: Entity.

  Returns:
    The new value for the specified property which Profile key.
  """
  reference_key = model_property.get_value_for_datastore(entity)

  if not reference_key:
    return None
  elif reference_key.kind() not in [GSoCProfile.kind(), GCIProfile.kind()]:
    raise ValueError(
        'Invalid kind %s for property %s',
            (reference_key.kind(), model_property.name))
  else:
    return _newKey(reference_key)


def _convertListProperty(model_property, entity):
  """Converts the specified ListProperty whose values are keys of GSoCProfile
  or GCIProfile type.

  Args:
    model_property: Property instance.
    entity: Entity.

  Returns:
    The new value for the specified property which is a list of Profile keys.
  """
  return [
      _newKey(old_key)
      for old_key in model_property.get_value_for_datastore(entity) or []]


def _convertParent(entity, parent=None):
  """Clones the specified entity, i.e. a new entity is created, and replaces
  its parent to either the specified one or a newly constructed one.

  If parent is not specified, it is assumed that the current parent of
  the specified entity is GSoCProfile or GCIProfile. A new one is constructed
  for the corresponding Profile.

  Args:
    entity: The specified DB entity.
    parent: Optional parent DB key.

  Returns:
    The newly created entity.
  """
  properties = dict(
      (k, v.get_value_for_datastore(entity))
      for k, v in entity.__class__.properties().iteritems())

  if not parent:
    parent = _newKey(entity.parent_key())
  properties.update(parent=parent)

  new_entity = entity.__class__(**properties)
  return new_entity


def _convertNDBParent(entity, parent=None):
  """Clones the specified entity, i.e. a new entity is created, and replaces
  its parent to either the specified one or a newly constructed one.

  If parent is not specified, it is assumed that the current parent of
  the specified entity is GSoCProfile or GCIProfile. A new one is constructed
  for the corresponding Profile.

  Args:
    entity: The specified NDB entity.
    parent: Optional parent NDB key.

  Returns:
    The newly created entity.
  """
  properties = entity.to_dict()
  if not parent:
    parent = _newProfileNDBKey(entity.key.parent())
  properties.update(parent=parent)

  new_entity = entity.__class__(**properties)
  return new_entity


@db.transactional(xg=True)
def convertGSoCProfileDBEntityGroup(profile_key):
  """Converts DB based part of entity group associated with the specified
  profile.

  Args:
    profile_key: db.Key of the profile to process
  """
  # map that associate old keys with new ones which are created during
  # the conversion
  conversion_map = {}
  to_delete = []
  do_put = True

  proposals = GSoCProposal.all().ancestor(profile_key).fetch(1000)
  for proposal in proposals:
    # update GSoCProposal.parent
    new_proposal = _convertParent(proposal)

    # update GSoCProposal.possible_mentors
    new_proposal.possible_mentors = _convertListProperty(
        GSoCProposal.possible_mentors, new_proposal)

    # update GSoCProposal.mentor
    new_proposal.mentor = _convertReferenceProperty(
        GSoCProposal.mentor, new_proposal)
    to_delete.append(proposal)
    if do_put:
      new_proposal.put()
      conversion_map[proposal.key()] = new_proposal.key()

    comments = GSoCComment.all().ancestor(proposal).fetch(1000)
    for comment in comments:
      # update GSoCComment.parent
      new_comment = _convertParent(comment, parent=new_proposal.key())

      # update GSoCComment.author
      new_comment.author = _convertReferenceProperty(
          GSoCComment.author, new_comment)
      if do_put:
        new_comment.put()
      to_delete.append(comment)

    scores = GSoCScore.all().ancestor(proposal).fetch(1000)
    for score in scores:
      # update GSoCScore.parent
      new_score = _convertParent(score, parent=new_proposal.key())

      # update GSoCScore.author
      new_score.author = _convertReferenceProperty(GSoCScore.author, new_score)
      if do_put:
        new_score.put()
      to_delete.append(score)

  projects = GSoCProject.all().ancestor(profile_key).fetch(1000)
  for project in projects:
    # update GSoCProject.parent
    new_project = _convertParent(project)

    # update GSoCProject.mentors
    new_project.mentors = _convertListProperty(GSoCProject.mentors, new_project)

    # update GSoCProject.proposal
    proposal_key = GSoCProject.proposal.get_value_for_datastore(project)
    if proposal_key:
      new_project.proposal = conversion_map.get(
          GSoCProject.proposal.get_value_for_datastore(project))

    if do_put:
      new_project.put()
      conversion_map[project.key()] = new_project.key()
    to_delete.append(project)

    grading_records = GSoCGradingRecord.all().ancestor(project.key())
    for grading_record in grading_records:
      # update GSoCGradingProjectSurveyRecord.project
      # this is another entity group, but XG transaction does the thing
      grading_project_survey_record_key = (
          GSoCGradingRecord.mentor_record.get_value_for_datastore(
              grading_record))
      if grading_project_survey_record_key:
        grading_project_survey_record = GSoCGradingProjectSurveyRecord.get(
            grading_project_survey_record_key)
        if grading_project_survey_record:
          grading_project_survey_record.project = new_project.key()
          if do_put:
            grading_project_survey_record.put()

      # update GSoCProjectSurveyRecord.project
      # this is another entity group, but XG transaction does the thing
      project_survey_record_key = (
          GSoCGradingRecord.student_record.get_value_for_datastore(
              grading_record))
      if project_survey_record_key:
        project_survey_record = GSoCProjectSurveyRecord.get(
            project_survey_record_key)
        if project_survey_record:
          project_survey_record.project = new_project.key()
          if do_put:
            project_survey_record.put()

      # update GSoCGradingRecord.parent
      new_grading_record = _convertParent(
          grading_record, parent=new_project.key())
      if do_put:
        new_grading_record.put()

    code_samples = GSoCCodeSample.all().ancestor(project.key())
    for code_sample in code_samples:
      # update GSoCCodeSample.parent
      new_code_sample = _convertParent(code_sample, parent=new_project.key())
      if do_put:
        new_code_sample.put()
      to_delete.append(code_sample)

  db.delete(to_delete)


@ndb.transactional
def convertGSoCProfileNDBEntityGroup(profile_key):
  """Converts NDB based part of entity group associated with the specified
  profile.

  Args:
    profile_key: db.Key of the profile to process
  """
  # NOTE: profile_key will always be an instance of db.Key because NDB is not
  # supported by MapReduce API.
  profile_key = ndb.Key.from_old_key(profile_key)

  to_delete = []
  do_put = True

  extensions = survey_model.PersonalExtension.query(ancestor=profile_key)
  for extension in extensions:
    # update PersonalExtension.parent
    new_extension = _convertNDBParent(extension)
    if do_put:
      new_extension.put()
    to_delete.append(extension)


@db.transactional
def convertGCIProfileDBEntityGroup(profile_key):
  """Converts DB based part of entity group associated with the specified
  profile.

  Args:
    profile_key: db.Key of the profile to process.
  """
  to_delete = []
  do_put = True

  org_scores = GCIOrgScore.all().ancestor(profile_key).fetch(1000)
  for org_score in org_scores:
    new_org_score = _convertParent(org_score)
    if do_put:
      new_org_score.put()
    to_delete.append(org_score)

  scores = GCIScore.all().ancestor(profile_key).fetch(1000)
  for score in scores:
    new_score = _convertParent(score)
    if do_put:
      new_score.put()
    to_delete.append(score)

  db.delete(to_delete)


@ndb.transactional
def convertGCIProfileNDBEntityGroup(profile_key):
  """Converts NDB based part of entity group associated with the specified
  profile.

  Args:
    profile_key: db.Key of the profile to process.
  """
  # NOTE: profile_key will always be an instance of db.Key because NDB is not
  # supported by MapReduce API.
  profile_key = ndb.Key.from_old_key(profile_key)

  to_delete = []
  do_put = True

  connections = connection_model.Connection.query(
      ancestor=profile_key).fetch(1000)
  for connection in connections:
    # update Connection.parent
    new_connection = _convertNDBParent(connection)
    if do_put:
      new_connection.put()
      to_delete.append(connection.key)

    messages = connection_model.ConnectionMessage.query(
        ancestor=connection.key).fetch(1000)
    for message in messages:
      new_message = _convertNDBParent(message, parent=new_connection.key)
      if do_put:
        new_message.put()
        to_delete.append(message.key)

  ndb.delete_multi(to_delete)


@db.transactional
def convertGCITask(task_key):
  """Converts the specified task by changing values of its profile related
  properties to the new NDB based profile entities.

  Args:
    task_key: Task key.
  """
  task = GCITask.get(task_key)
  task.created_by = _convertReferenceProperty(GCITask.created_by, task)
  task.modified_by = _convertReferenceProperty(GCITask.modified_by, task)
  task.student = _convertReferenceProperty(GCITask.student, task)
  task.mentors = _convertListProperty(GCITask.mentors, task)
  task.subscribers = _convertListProperty(GCITask.subscribers, task)
  task.put()


@db.transactional
def convertGCIOrg(org_key):
  """Converts the specified organization by changing values of its profile
  related properties to the new NDB based profile entities.

  Args:
    org_key: Organization key.
  """
  org = GCIOrganization.get(org_key)
  org.proposed_winners = _convertListProperty(
      GCIOrganization.proposed_winners, org)
  org.backup_winner = _convertReferenceProperty(
      GCIOrganization.backup_winner, org)
  org.put()


@db.transactional
def convertGCIBulkCreateData(bulk_create_data_key):
  """Converts the specified bulk create data by changing values of its profile
  related properties to the new NDB based profile entities.

  Args:
    org_key: BulkCreateData key.
  """
  bulk_create_data = GCIBulkCreateData.get(bulk_create_data_key)
  bulk_create_data.created_by = _convertReferenceProperty(
      GCIBulkCreateData.created_by, bulk_create_data)
  bulk_create_data.put()


def counter(entity_key):
  """Mapper that simply counts entities of the specified model.

  Args:
    entity_key: Entity key.
  """
  yield operation.counters.Increment('counter')