| # Copyright 2015 the Melange authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Map-reduce scripts to update grading record entities.""" |
| |
| import logging |
| |
| from melange.appengine import django_setup |
| django_setup.setup_environment() |
| |
| from google.appengine.ext import db |
| from google.appengine.ext import ndb |
| |
| from mapreduce import operation |
| |
| # MapReduce requires import of processed model classes. |
| # pylint: disable=unused-import |
| from soc.modules.gsoc.models.grading_record import GSoCGradingRecord |
| from soc.modules.gsoc.models.project import GSoCProject |
| from summerofcode.models.project import Project |
| # pylint: enable=unused-import |
| |
| |
| def deleteDuplicateRecords(record_key): |
| """Deletes old records which are duplicated in the datastore. |
| |
| Args: |
| record_key: db.Key of GSoCGradingRecord entity. |
| """ |
| dry_run = True |
| project_key = record_key.parent() |
| |
| if project_key.kind() == 'GSoCProject': |
| project = db.get(project_key) |
| if not project: |
| yield operation.counters.Increment('No project') |
| |
| # It is likely that the key has form |
| # (User, user_id), (GSoCProfile, profile_id), (GSoCProject, project_id) |
| # The problem is that in the meantime GSoCProfile entities were |
| # updated so that they reflect that GSoCProfile was converted to Profile. |
| # Therefore, the structure is |
| # (User, user_id), (Profile, profile_id), (GSoCProject, project_id') |
| # We can find corresponding GSoCGradingRecord entities because all data |
| # was converted at that point. |
| # It is not trivial to find those entities to verify they exist but we |
| # can rely on the fact that each student had at most one project |
| old_profile_key = project_key.parent() # the old GSoCProfile key |
| new_profile_key = db.Key.from_path( |
| 'Profile', old_profile_key.name(), parent=old_profile_key.parent()) |
| projects = GSoCProject.all().ancestor(new_profile_key).fetch(1000) |
| if not projects: |
| logging.error('No projects for profile %s', new_profile_key) |
| yield operation.counters.Increment('No new project') |
| elif len(projects) > 1: |
| logging.error('More than one project for profile %s', new_profile_key) |
| yield operation.counters.Increment('More than one new project') |
| else: |
| project = projects[0] |
| |
| old_grading_record = db.get(record_key) |
| survey_group_key = ( |
| GSoCGradingRecord.grading_survey_group .get_value_for_datastore( |
| old_grading_record)) |
| |
| # find corresponding GSoCGradingRecords |
| grading_records = ( |
| GSoCGradingRecord |
| .all() |
| .filter('grading_survey_group', survey_group_key) |
| .ancestor(project) |
| .fetch(1000)) |
| if not grading_records: |
| logging.error('No grading record for %s', project.key()) |
| yield operation.counters.Increment('No grading record') |
| elif len(grading_records) > 1: |
| logging.error('More than one grading records for %s', project.key()) |
| yield operation.counters.Increment('More than one grading record') |
| else: |
| grading_record = grading_records[0] |
| |
| # make sure that all properties are the same |
| property_names = GSoCGradingRecord.properties().keys() |
| wrong = False |
| for property_name in property_names: |
| if property_name == 'modified': |
| continue |
| |
| old = getattr(old_grading_record, property_name) |
| new = getattr(grading_record, property_name) |
| |
| # compare keys for reference properties |
| if property_name in ( |
| 'grading_survey_group', 'mentor_record', 'student_record'): |
| if old and not new: |
| logging.error('old != new, %s != %s', old.key(), new) |
| wrong = True |
| elif not old and new: |
| logging.error('old != new, %s != %s', old, new.key()) |
| wrong = True |
| elif not old and not new: |
| continue |
| elif old.key() != new.key(): |
| logging.error('old != new, %s != %s', old.key(), new.key()) |
| wrong = True |
| # compare values for the rest of properties |
| elif old != new: |
| logging.error('old != new, %s != %s', old, new) |
| wrong = True |
| |
| if wrong: |
| yield operation.counters.Increment('Different values') |
| else: |
| # when grading_record was created modified property was marked with |
| # auto_now=True. Now it is time to update it. |
| # auto_now is effective on every put operation so the model must |
| # be modified while this operation runs |
| # GSoCGradingRecord.modified.auto_now = False |
| grading_record.modified = old_grading_record.modified |
| if not dry_run: |
| grading_record.put() |
| |
| # everything is OK. there is a newer entity with the same data |
| # we can get rid of the old entity |
| if not dry_run: |
| old_grading_record.delete() |
| yield operation.counters.Increment('Successfully processed') |
| |
| |
| def verifyRecordsHasProject(record_key): |
| """Verifies that the specified record correspond to an existing project |
| |
| Args: |
| record_key: db.Key of GSoCGradingRecord entity. |
| """ |
| project_key = record_key.parent() |
| if project_key.kind() == 'GSoCProject': |
| project = db.get(project_key) |
| if not project: |
| yield operation.counters.Increment('No DB project') |
| elif project_key.kind() == 'Project': |
| project = ndb.Key.from_old_key(project_key).get() |
| if not project: |
| yield operation.counters.Increment('No NDB project') |
| else: |
| yield operation.counters.Increment('Unknown kind: %s' % project_key.kind()) |
| |
| |
| def updateProject(record_key): |
| """Updates project corresponding to the specified record. |
| |
| If the project has 'GSoCProject' kind, a new entity is created and its kind |
| is 'Project'. |
| |
| Args: |
| record_key: db.Key of GSoCGradingRecord entity. |
| """ |
| dry_run = True |
| |
| project_key = record_key.parent() |
| if project_key.kind() == 'GSoCProject': |
| yield operation.counters.Increment('GSoCProject kind') |
| |
| project = db.get(project_key) |
| if not project.new_project: |
| logging.error('No new project for %s', project_key) |
| yield operation.counters.Increment('No new project') |
| else: |
| record = db.get(record_key) |
| new_project_key = db.Key(encoded=project.new_project) |
| |
| properties = { |
| 'grading_survey_group': ( |
| GSoCGradingRecord |
| .grading_survey_group |
| .get_value_for_datastore(record)), |
| 'mentor_record': ( |
| GSoCGradingRecord |
| .mentor_record |
| .get_value_for_datastore(record)), |
| 'student_record': ( |
| GSoCGradingRecord |
| .student_record |
| .get_value_for_datastore(record)), |
| 'grade_decision': record.grade_decision, |
| 'locked': record.locked, |
| 'created': record.created, |
| 'modified': record.modified, |
| 'parent': new_project_key, |
| } |
| new_record = GSoCGradingRecord(**properties) |
| |
| if not dry_run: |
| new_record.put() |
| record.delete() |
| yield operation.counters.Increment('New record created') |