app/soc/mapreduce/convert_grading_record.py

# Copyright 2015 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Map-reduce scripts to update grading record entities."""

import logging

from melange.appengine import django_setup
django_setup.setup_environment()

from google.appengine.ext import db
from google.appengine.ext import ndb

from mapreduce import operation

# MapReduce requires import of processed model classes.
# pylint: disable=unused-import
from soc.modules.gsoc.models.grading_record import GSoCGradingRecord
from soc.modules.gsoc.models.project import GSoCProject
from summerofcode.models.project import Project
# pylint: enable=unused-import


def deleteDuplicateRecords(record_key):
  """Deletes old records which are duplicated in the datastore.

  Args:
    record_key: db.Key of GSoCGradingRecord entity.
  """
  dry_run = True
  project_key = record_key.parent()

  if project_key.kind() == 'GSoCProject':
    project = db.get(project_key)
    if not project:
      yield operation.counters.Increment('No project')

      # It is likely that the key has form
      # (User, user_id), (GSoCProfile, profile_id), (GSoCProject, project_id)
      # The problem is that in the meantime GSoCProfile entities were
      # updated so that they reflect that GSoCProfile was converted to Profile.
      # Therefore, the structure is
      # (User, user_id), (Profile, profile_id), (GSoCProject, project_id')
      # We can find corresponding GSoCGradingRecord entities because all data
      # was converted at that point.
      # It is not trivial to find those entities to verify they exist but we
      # can rely on the fact that each student had at most one project
      old_profile_key = project_key.parent()  # the old GSoCProfile key
      new_profile_key = db.Key.from_path(
          'Profile', old_profile_key.name(), parent=old_profile_key.parent())
      projects = GSoCProject.all().ancestor(new_profile_key).fetch(1000)
      if not projects:
        logging.error('No projects for profile %s', new_profile_key)
        yield operation.counters.Increment('No new project')
      elif len(projects) > 1:
        logging.error('More than one project for profile %s', new_profile_key)
        yield operation.counters.Increment('More than one new project')
      else:
        project = projects[0]

        old_grading_record = db.get(record_key)
        survey_group_key = (
            GSoCGradingRecord.grading_survey_group .get_value_for_datastore(
                old_grading_record))

        # find corresponding GSoCGradingRecords
        grading_records = (
            GSoCGradingRecord
            .all()
            .filter('grading_survey_group', survey_group_key)
            .ancestor(project)
            .fetch(1000))
        if not grading_records:
          logging.error('No grading record for %s', project.key())
          yield operation.counters.Increment('No grading record')
        elif len(grading_records) > 1:
          logging.error('More than one grading records for %s', project.key())
          yield operation.counters.Increment('More than one grading record')
        else:
          grading_record = grading_records[0]

          # make sure that all properties are the same
          property_names = GSoCGradingRecord.properties().keys()
          wrong = False
          for property_name in property_names:
            if property_name == 'modified':
              continue

            old = getattr(old_grading_record, property_name)
            new = getattr(grading_record, property_name)

            # compare keys for reference properties
            if property_name in (
                'grading_survey_group', 'mentor_record', 'student_record'):
              if old and not new:
                logging.error('old != new, %s != %s', old.key(), new)
                wrong = True
              elif not old and new:
                logging.error('old != new, %s != %s', old, new.key())
                wrong = True
              elif not old and not new:
                continue
              elif old.key() != new.key():
                logging.error('old != new, %s != %s', old.key(), new.key())
                wrong = True
            # compare values for the rest of properties
            elif old != new:
              logging.error('old != new, %s != %s', old, new)
              wrong = True

          if wrong:
            yield operation.counters.Increment('Different values')
          else:
            # when grading_record was created modified property was marked with
            # auto_now=True. Now it is time to update it.
            # auto_now is effective on every put operation so the model must
            # be modified while this operation runs
            # GSoCGradingRecord.modified.auto_now = False
            grading_record.modified = old_grading_record.modified
            if not dry_run:
              grading_record.put()

            # everything is OK. there is a newer entity with the same data
            # we can get rid of the old entity
            if not dry_run:
              old_grading_record.delete()
            yield operation.counters.Increment('Successfully processed')


def verifyRecordsHasProject(record_key):
  """Verifies that the specified record correspond to an existing project

  Args:
    record_key: db.Key of GSoCGradingRecord entity.
  """
  project_key = record_key.parent()
  if project_key.kind() == 'GSoCProject':
    project = db.get(project_key)
    if not project:
      yield operation.counters.Increment('No DB project')
  elif project_key.kind() == 'Project':
    project = ndb.Key.from_old_key(project_key).get()
    if not project:
      yield operation.counters.Increment('No NDB project')
  else:
    yield operation.counters.Increment('Unknown kind: %s' % project_key.kind())


def updateProject(record_key):
  """Updates project corresponding to the specified record.

  If the project has 'GSoCProject' kind, a new entity is created and its kind
  is 'Project'.

  Args:
    record_key: db.Key of GSoCGradingRecord entity.
  """
  dry_run = True

  project_key = record_key.parent()
  if project_key.kind() == 'GSoCProject':
    yield operation.counters.Increment('GSoCProject kind')

    project = db.get(project_key)
    if not project.new_project:
      logging.error('No new project for %s', project_key)
      yield operation.counters.Increment('No new project')
    else:
      record = db.get(record_key)
      new_project_key = db.Key(encoded=project.new_project)

      properties = {
          'grading_survey_group': (
              GSoCGradingRecord
              .grading_survey_group
              .get_value_for_datastore(record)),
          'mentor_record': (
              GSoCGradingRecord
              .mentor_record
              .get_value_for_datastore(record)),
          'student_record': (
              GSoCGradingRecord
              .student_record
              .get_value_for_datastore(record)),
          'grade_decision': record.grade_decision,
          'locked': record.locked,
          'created': record.created,
          'modified': record.modified,
          'parent': new_project_key,
          }
      new_record = GSoCGradingRecord(**properties)

      if not dry_run:
        new_record.put()
        record.delete()
      yield operation.counters.Increment('New record created')