diff --git a/app/model/index.js b/app/model/index.js
index 9aef8014f..10131e188 100644
--- a/app/model/index.js
+++ b/app/model/index.js
@@ -15,7 +15,6 @@ exports.news = require('./news');
exports.playlists = require('./playlists');
exports.templates = require('./templates');
exports.soundscapes = require('./soundscapes');
-exports.uploads = require('./uploads');
exports.orders = require('./orders');
exports.settings = require('./system-settings');
exports.classifications = require('./classifications');
diff --git a/app/routes/data-api/index.js b/app/routes/data-api/index.js
index d78620866..6a85e6c1a 100644
--- a/app/routes/data-api/index.js
+++ b/app/routes/data-api/index.js
@@ -10,7 +10,6 @@ var routes = [
'/orders',
'/jobs',
'/app-listings',
- '/uploads'
];
diff --git a/app/routes/data-api/project/index.js b/app/routes/data-api/project/index.js
index 3edaff534..8c7854175 100644
--- a/app/routes/data-api/project/index.js
+++ b/app/routes/data-api/project/index.js
@@ -564,6 +564,5 @@ router.use('/:projectUrl/tags', tagRoutes);
router.use('/:projectUrl/audio-event-detections', require('./audio-event-detections'));
router.use('/:projectUrl/soundscape-composition', require('./soundscape-composition'));
router.use('/:projectUrl/citizen-scientist', require('./citizen-scientist'));
-router.use('/:projectUrl/uploads', require('./uploads'));
module.exports = router;
diff --git a/app/routes/data-api/project/uploads.js b/app/routes/data-api/project/uploads.js
deleted file mode 100644
index 54b657c52..000000000
--- a/app/routes/data-api/project/uploads.js
+++ /dev/null
@@ -1,50 +0,0 @@
-var debug = require('debug')('arbimon2:route:uploads');
-var router = require('express').Router();
-
-var model = require('../../../model');
-
-router.get('/processing', function(req, res, next) {
- res.type('json');
-
- model.uploads.getUploadingRecordings({
- project: req.project.project_id,
- site: req.query.site
- }).then(function(uploads){
- res.json(uploads);
- }, next);
-});
-
-router.get('/check', function(req, res, next) {
- res.type('json');
-
- checkStatus(req, res, next);
-});
-
-async function checkStatus(req, res, next) {
- const idToken = req.session.idToken
- const queryItems = Array.isArray(req.query.items) ? req.query.items : [req.query.items]
- const items = queryItems.map(i => { return JSON.parse(i) })
- for (let item of items) {
- if (!item.uploadUrl) return
- const code = await model.uploads.checkStatus(item.uploadUrl, idToken)
- if (code) {
- const status = getStatus(code)
- item.status = status
- await model.uploads.updateStateAsync({ uploadId: item.id, status: status, uploadUrl: item.uploadUrl })
- }
- }
- res.json({ items: items });
-}
-
-const getStatus = function (code) {
- switch (code) {
- case 0: return 'waiting';
- case 10: return 'waiting';
- case 20: return 'uploaded';
- case 30: return 'error';
- case 32: return 'error';
- default: return 0
- }
-}
-
-module.exports = router;
diff --git a/app/routes/data-api/uploads.js b/app/routes/data-api/uploads.js
deleted file mode 100644
index 23dddcb37..000000000
--- a/app/routes/data-api/uploads.js
+++ /dev/null
@@ -1,202 +0,0 @@
-var fs = require('fs');
-var express = require('express');
-var router = express.Router();
-var async = require('async');
-var util = require('util');
-var _ = require('lodash');
-var q = require('q');
-
-const fileHelper = require('../../utils/file-helper')
-var model = require('../../model');
-var tmpFileCache = require('../../utils/tmpfilecache');
-var formatParse = require('../../utils/format-parse');
-const uploadQueue = require('../../utils/upload-queue');
-
-var deleteFile = function(filename) {
- fs.unlink(filename, function(err) {
- if(err) console.error("failed to delete: %s", filename);
- });
-};
-
-// middleware to handle upload auth
-var authorize = function(authtype){
- return function(req, res, next) {
- // if(req.systemSettings('feature.uploads') == 'off') {
- // return res.status(503).json({ error: 'uploads are unavailable, try again later' });
- // }
- console.log('--authorize', req.query.project)
- var accessToken = req.get('X-X-access-token-X-X') || req.body.token;
-
- if(authtype.session && req.session && req.session.loggedIn) {
- if(!req.query.project || !req.query.site || !req.query.nameformat || !req.query.timezone) {
- console.error('--missing parameters', req.query.project, req.query.site, req.query.nameformat, req.query.timezone)
- return res.status(400).json({ error: 'missing parameters' });
- }
-
- console.log('project_id: %s | site_id: %s |format: %s |timezone: %s',
- req.query.project,
- req.query.site,
- req.query.nameformat,
- req.query.timezone
- );
-
- var perm = 'manage project recordings';
-
- if(!req.haveAccess(req.query.project, perm)) {
- console.error(`you dont have permission to ${ perm }`)
- res.status(401).json({
- error: `you dont have permission to ${ perm }`
- });
- return;
- }
-
- req.upload = {
- userId: req.session.user.id,
- projectId: Number(req.query.project),
- siteId: Number(req.query.site),
- nameFormat: req.query.nameformat,
- timezone: req.query.timezone,
- };
-
- next();
- }
-
- // verify token
- else if(authtype.token && req.token) {
-
- req.upload = {
- userId: 0,
- projectId: Number(req.token.project),
- siteId: Number(req.token.site),
- nameFormat: 'Arbimon',
- timezone: 'local',
- };
-
- next();
- }
-
- // verify access token
- else if(authtype.access_token && accessToken) {
- res.type('json');
- return model.AccessTokens.verifyTokenAccess(accessToken, 'manage project recordings', {requireScope:true, requireProject:true}).then(function(resolvedToken){
- return model.users.hasProjectAccess(resolvedToken.user, resolvedToken.project, {required:true}).then(function(){
- req.upload = {
- userId: resolvedToken.user,
- projectId: Number(resolvedToken.project),
- siteId: Number(resolvedToken.site),
- nameFormat: resolvedToken.nameFormat || 'any',
- timezone: resolvedToken.timezone || 'local',
- };
- });
- }).finally(next);
- }
- else {
- // error not logged user nor site token
- return res.sendStatus(401);
- }
- };
-};
-
-var verifySite = function(req, res, next) {
- model.sites.findById(req.upload.siteId, function(err, rows) {
- if(err) return next(err);
-
- if(!rows.length) {
- return res.status(403).json({ error: "site does not exist" });
- }
-
- var site = rows[0];
- console.log('--site to upload', site)
- if(site.project_id !== req.upload.projectId) {
- return res.status(403).json({ error: "site does not belong to project"});
- }
-
- next();
- });
-};
-
-var receiveUpload = function(req, res, next) {
- res.type('json');
-
- var upload = {};
- var error = false;
-
- if(!req.busboy) return res.status(400).json({ error: "no data" });
-
- req.busboy.on('field', function(fieldname, val) {
- console.log('field %s = %s', fieldname, val);
- });
-
- req.busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
- console.log('file fieldname: %s | filename: %s', fieldname, filename);
-
- // FFI -> filename format info
- try {
- upload.FFI = formatParse(req.upload.nameFormat, filename);
- }
- catch(e) {
- file.resume();
- error = true;
- return res.status(400).json({ error: e.message });
- }
-
- var saveTo = tmpFileCache.key2File(Date.now()+filename);
- // ^__ concat now() to prevent collitions
- file.pipe(fs.createWriteStream(saveTo));
-
- upload.name = filename;
- upload.path = saveTo;
- upload.timezone = req.upload.timezone
-
- });
-
- req.busboy.on('finish', function() {
- if(error) return;
-
- console.log('filename: ', upload.name);
-
- async.waterfall([
- function recNotExists(callback) {
- model.recordings.exists({
- site_id: req.upload.siteId,
- filename: upload.FFI.filename
- }, function(err, exists) {
- if(err) return next(err);
-
- if(exists) {
- deleteFile(upload.path);
- var msg = "filename "+ upload.FFI.filename +
- " already exists on site " + req.upload.siteId;
- return res.status(403).json({ error: msg });
- }
-
- callback();
- });
- },
- function sendToProcess(callback) {
- upload.projectId = req.upload.projectId;
- upload.siteId = req.upload.siteId;
- upload.userId = req.upload.userId;
- const idToken = req.session.idToken
- const uploadsBody = {
- originalFilename: upload.name,
- filePath: upload.path,
- fileExt: fileHelper.getExtension(upload.name),
- streamId: upload.siteId
- }
- uploadQueue.enqueue(upload, uploadsBody, idToken, function(err) {
- if(err) return next(err);
-
- res.status(202).json({ success: "upload processing" });
- });
- }
- ]);
-
- });
-
- req.pipe(req.busboy);
-};
-
-router.post('/audio', authorize({session:true, token:true, access_token:true}), verifySite, receiveUpload);
-
-module.exports = router;
diff --git a/app/utils/upload-queue.js b/app/utils/upload-queue.js
deleted file mode 100644
index 4f0bcfce2..000000000
--- a/app/utils/upload-queue.js
+++ /dev/null
@@ -1,74 +0,0 @@
-/*jshint node:true */
-"use strict";
-
-/**
- @module utils/upload-queue
- @description
- Module export async.queue
- using as worker processUpload()
- @example
- var uploadQueue = require('../utils/upload-queue');
-
- uploadQueue.enqueue(upload); // processUpload() upload param
- */
-var debug = require('debug')('arbimon2:upload-queue');
-var async = require('async');
-var model = require('../model');
-const moment = require('moment-timezone');
-const fileHelper = require('../utils/file-helper')
-
-
-module.exports = {
- enqueue: function(upload, uploadsBody, idToken, cb) {
- var upload_row = {
- filename: upload.name,
- project_id: upload.projectId,
- site_id: upload.siteId,
- user_id: upload.userId,
- state: 'initializing',
- datetime: upload.FFI.datetime
- }
- async.waterfall([
- function(callback) {
- model.sites.getSiteTimezone(upload.siteId, callback)
- },
- function(timezone, callback) {
- const isLocal = upload.timezone === 'local'
- // Convert datetime with timezone offsets for browser AudioMoth recordings
- // https://github.com/rfcx/arbimon/commit/efa1a487ce672ecf3d81c45470511e3f46a69305
- if (upload.info && upload.info.isUTC) {
- const datetimeLocal = timezone ? moment.tz(upload_row.datetime, timezone).toISOString() : null;
- upload_row.datetime = datetimeLocal? datetimeLocal : upload_row.datetime;
- } else {
- const datetimeUtc = moment.utc(upload_row.datetime).format('YYYY-MM-DD HH:mm:ss')
- upload_row.datetime = isLocal ? upload_row.datetime : datetimeUtc;
- }
- const timezoneOffset = fileHelper.tzOffsetMinutesFromTzName(isLocal ? timezone : 'UTC')
- uploadsBody.timestamp = moment(upload.FFI.datetime).utcOffset(timezoneOffset, true)
- model.uploads.insertRecToList(upload_row, callback);
- },
- function(result, fields, callback) {
- upload.id = result.insertId;
- upload_row.upload_id = result.insertId;
- model.sites.getSiteExternalId(upload_row.site_id, callback)
- },
- function storeRawFileInBucket(externalId, callback) {
- uploadsBody.streamId = externalId ? externalId : null
- model.uploads.uploadFile(uploadsBody, idToken, callback);
- },
- function flagAsWaiting(uploadId, callback) {
- upload.uploadId = uploadId
- model.uploads.updateState({ uploadId: upload.id, status: 'waiting', uploadUrl: uploadId }, function(err){
- callback(err);
- });
- }
- ], cb);
- },
- resume: function(){
- return model.uploads.fetchRandomUploadItems(10).then(function(upload_items){
- if(upload_items.length){
- scheduler.push.apply(scheduler, upload_items.map(uploadFromUploadItemEntry));
- }
- }).catch(console.error);
- }
-};
diff --git a/assets/app/app/audiodata/index.js b/assets/app/app/audiodata/index.js
index 33a7c4f8f..41511d134 100644
--- a/assets/app/app/audiodata/index.js
+++ b/assets/app/app/audiodata/index.js
@@ -5,7 +5,6 @@ angular.module('a2.audiodata', [
'a2.directive.audio-bar',
'a2.audiodata.sites',
'a2.audiodata.species',
- 'a2.audiodata.uploads',
'a2.audiodata.recordings',
'a2.audiodata.training-sets',
'a2.audiodata.playlists',
diff --git a/assets/app/app/audiodata/uploads/index.js b/assets/app/app/audiodata/uploads/index.js
deleted file mode 100644
index bbad3bc14..000000000
--- a/assets/app/app/audiodata/uploads/index.js
+++ /dev/null
@@ -1,13 +0,0 @@
-angular.module('a2.audiodata.uploads', [
- 'ui.router',
- 'a2.audiodata.uploads.upload',
- 'a2.audiodata.uploads.processing',
-])
-.config(function($stateProvider, $urlRouterProvider) {
- $stateProvider.state('audiodata.uploads', {
- url: '/uploads',
- template:'
Name | -Size | -Progress | -Status | -Actions | -
---|---|---|---|---|
{{ item.file.name }} | -{{ item.file.size | prettyBytes }} | -
-
-
-
- |
- - Uploading - Success - - Error - - - - Duplicate - - - - | -- - - | -
Upload 1,000 files at a time
-Max 200 Mb per file
-