Skip to content

Commit

Permalink
Migrate data_preprocessing script from repo (#272)
Browse files Browse the repository at this point in the history 
* Migrate data_preprocessing script from repo

---------

Co-authored-by: aescobarr <agustiescobar@gmail.com>
  • Loading branch information
@epou @aescobarr
epou and aescobarr authored Jan 7, 2025
1 parent 029dd18 commit a985611
Show file tree
Hide file tree
Showing 5 changed files with 2,054 additions and 23 deletions.
37 changes: 34 additions & 3 deletions tigaserver_app/serializers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ class Meta:
'location_choice', 'current_location_lon', 'current_location_lat', 'mission',
'selected_location_lon', 'selected_location_lat', 'note', 'package_name', 'package_version',
'device_manufacturer', 'device_model', 'os', 'os_version', 'os_language', 'app_language', 'hide',
'type','creation_time')
'type','creation_time', 'timezone')


class MapDataSerializer(serializers.ModelSerializer):
Expand Down Expand Up @@ -562,7 +562,38 @@ def get_country(self,obj):

class Meta:
model = Report
exclude = ('version_number', 'user', 'report_id', 'server_upload_time', 'phone_upload_time', 'version_time', 'location_choice', 'current_location_lon', 'current_location_lat', 'mission', 'selected_location_lon', 'selected_location_lat', 'note', 'package_name', 'package_version', 'device_manufacturer', 'device_model', 'os', 'os_version', 'os_language', 'app_language', 'hide')
fields = (
'version_UUID',
'creation_time',
'creation_date',
'creation_day_since_launch',
'creation_year',
'creation_month',
'site_cat',
'type',
'lon',
'lat',
'movelab_annotation',
'movelab_annotation_euro',
'tiger_responses',
'tiger_responses_text',
'site_responses',
'site_responses_text',
'tigaprob_cat',
'latest_version',
'visible',
'n_photos',
'final_expert_status_text',
'responses',
'country',
'updated_at',
'datetime_fix_offset',
'point',
'nuts_2',
'nuts_3',
'cached_visible',
'session',
)


class SiteMapSerializer(serializers.ModelSerializer):
Expand All @@ -576,7 +607,7 @@ class SiteMapSerializer(serializers.ModelSerializer):

class Meta:
model = Report
exclude = ('version_UUID', 'version_number', 'user', 'report_id', 'server_upload_time', 'phone_upload_time', 'version_time', 'location_choice', 'current_location_lon', 'current_location_lat', 'mission', 'selected_location_lon', 'selected_location_lat', 'note', 'package_name', 'package_version', 'device_manufacturer', 'device_model', 'os', 'os_version', 'os_language', 'app_language', 'hide')
exclude = ('version_UUID', 'version_number', 'user', 'report_id', 'server_upload_time', 'phone_upload_time', 'version_time', 'location_choice', 'current_location_lon', 'current_location_lat', 'mission', 'selected_location_lon', 'selected_location_lat', 'note', 'package_name', 'package_version', 'device_manufacturer', 'device_model', 'os', 'os_version', 'os_language', 'app_language', 'hide', 'timezone')


class CoverageMapSerializer(serializers.ModelSerializer):
Expand Down
25 changes: 5 additions & 20 deletions tigaserver_app/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1951,10 +1951,7 @@ def uuid_list_autocomplete(request):
@api_view(['GET'])
def all_reports_paginated(request):
if request.method == 'GET':
if conf.FAST_LOAD and conf.FAST_LOAD == True:
non_visible_report_id = []
else:
non_visible_report_id = [report.version_UUID for report in Report.objects.all() if not report.visible]
non_visible_report_id = [report.version_UUID for report in Report.objects.all() if not report.visible]
queryset = Report.objects.exclude(hide=True).exclude(type='mission').exclude(
version_UUID__in=non_visible_report_id).filter( package_filter )\
.exclude(package_name='ceab.movelab.tigatrapp', package_version=10).order_by('version_UUID')
Expand All @@ -1977,10 +1974,7 @@ def all_reports_internal(year):
@api_view(['GET'])
def all_reports(request):
if request.method == 'GET':
if conf.FAST_LOAD and conf.FAST_LOAD == True:
non_visible_report_id = []
else:
non_visible_report_id = [report.version_UUID for report in Report.objects.all() if not report.visible]
non_visible_report_id = [report.version_UUID for report in Report.objects.all() if not report.visible]
queryset = Report.objects.exclude(hide=True).exclude(type='mission').exclude(
version_UUID__in=non_visible_report_id).filter( package_filter )\
.exclude(package_name='ceab.movelab.tigatrapp', package_version=10)
Expand All @@ -2003,10 +1997,7 @@ def non_visible_reports_internal(year):
unfiltered_clean_reports_query = Report.objects.filter(version_UUID__in=unfiltered_clean_reports_id)

# new_reports_unfiltered_id = [ report.version_UUID for report in filtered_reports ]
if conf.FAST_LOAD and conf.FAST_LOAD == True:
non_visible_report_id = []
else:
non_visible_report_id = [report.version_UUID for report in
non_visible_report_id = [report.version_UUID for report in
Report.objects.exclude(version_UUID__in=unfiltered_clean_reports_id) if
not report.visible]

Expand Down Expand Up @@ -2040,10 +2031,7 @@ def non_visible_reports_paginated(request):
unfiltered_clean_reports_query = Report.objects.filter(version_UUID__in=unfiltered_clean_reports_id)

# new_reports_unfiltered_id = [ report.version_UUID for report in filtered_reports ]
if conf.FAST_LOAD and conf.FAST_LOAD == True:
non_visible_report_id = []
else:
non_visible_report_id = [report.version_UUID for report in
non_visible_report_id = [report.version_UUID for report in
Report.objects.exclude(version_UUID__in=unfiltered_clean_reports_id) if
not report.visible]

Expand Down Expand Up @@ -2079,10 +2067,7 @@ def non_visible_reports(request):
unfiltered_clean_reports_query = Report.objects.filter(version_UUID__in=unfiltered_clean_reports_id)

# new_reports_unfiltered_id = [ report.version_UUID for report in filtered_reports ]
if conf.FAST_LOAD and conf.FAST_LOAD == True:
non_visible_report_id = []
else:
non_visible_report_id = [report.version_UUID for report in
non_visible_report_id = [report.version_UUID for report in
Report.objects.exclude(version_UUID__in=unfiltered_clean_reports_id) if
not report.visible]

Expand Down
Loading

0 comments on commit a985611

Please sign in to comment.