Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Harvester / Remove records by harvester UUID #8431

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+99 −21
Draft

Harvester / Remove records by harvester UUID #8431

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 76 additions & 10 deletions domain/src/main/java/org/fao/geonet/repository/MetadataRepository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,20 +24,17 @@
package org.fao.geonet.repository;

import java.util.List;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import org.fao.geonet.domain.Metadata;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;

/**
* Data Access object for the {@link Metadata} entities.
*
* <p>
* The use of this class is discouraged, you should use IMetadataUtils or IMetadataManager instead.
*
* @author Jesse
Expand All @@ -60,12 +57,12 @@ public interface MetadataRepository extends GeonetRepository<Metadata, Integer>,

/**
* Find all metadata by the metadata's uuid.
*
* @param uuid the uuid of the metadata to find
* @return a list of metadata.
*/
@Nullable
List<Metadata> findAllByUuid(@Nonnull String uuid);
*
* @param uuid the uuid of the metadata to find
* @return a list of metadata.
*/
@Nullable
List<Metadata> findAllByUuid(@Nonnull String uuid);

/**
* Find all metadata harvested by the identified harvester.
Expand All @@ -76,7 +73,76 @@ public interface MetadataRepository extends GeonetRepository<Metadata, Integer>,
@Nonnull
List<Metadata> findAllByHarvestInfo_Uuid(@Nonnull String uuid);

int countByHarvestInfo_Uuid(@Nonnull String uuid);

@Query(value = "SELECT distinct(source) FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
List<String> findDistinctSourcesByHarvestInfo__uuid(@Param("harvesterUuid") String harvesterUuid);


@Query(value = "DELETE FROM operationallowed WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllOperationAllowedByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadatarating WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllMetadataRatingByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM validation WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllValidationByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM usersavedselections WHERE metadatauuid IN (SELECT uuid FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllUsersavedselectionsByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadatafiledownloads WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllMetadatafiledownloadsByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadatafileuploads WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllMetadatafileuploadsByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadatastatus WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllMetadatastatusByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadatalink WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllMetadatalinkByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadatacateg WHERE metadataid IN (SELECT id FROM metadata WHERE harvestuuid = :harvesterUuid)",
nativeQuery = true)
@Modifying
void deleteAllMetadatacategByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

@Query(value = "DELETE FROM metadata WHERE harvestuuid = :harvesterUuid",
nativeQuery = true)
@Modifying
void deleteAllMetadataByHarvesterUuid(@Param("harvesterUuid") String harvesterUuid);

default void deleteAllByHarvesterUuid(String harvesterUuid) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the reason for defining it as the default method?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

image

deleteAllOperationAllowedByHarvesterUuid(harvesterUuid);
deleteAllMetadataRatingByHarvesterUuid(harvesterUuid);
deleteAllValidationByHarvesterUuid(harvesterUuid);
deleteAllUsersavedselectionsByHarvesterUuid(harvesterUuid);
deleteAllMetadatafiledownloadsByHarvesterUuid(harvesterUuid);
deleteAllMetadatafiledownloadsByHarvesterUuid(harvesterUuid);
deleteAllMetadatafileuploadsByHarvesterUuid(harvesterUuid);
deleteAllMetadatastatusByHarvesterUuid(harvesterUuid);
deleteAllMetadatalinkByHarvesterUuid(harvesterUuid);
deleteAllMetadatacategByHarvesterUuid(harvesterUuid);
deleteAllMetadataByHarvesterUuid(harvesterUuid);
}

@Query(value = "SELECT replace(data, :search, :replace) FROM metadata m " +
"WHERE uuid = :uuid",
Expand Down
12 changes: 10 additions & 2 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/HarvestManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,15 @@
import org.fao.geonet.kernel.AccessManager;
import org.fao.geonet.kernel.DataManager;
import org.fao.geonet.kernel.HarvestInfoProvider;
import org.fao.geonet.kernel.datamanager.IMetadataManager;
import org.fao.geonet.kernel.harvest.Common.OperResult;
import org.fao.geonet.kernel.harvest.harvester.AbstractHarvester;
import org.fao.geonet.kernel.harvest.harvester.AbstractParams;
import org.fao.geonet.kernel.harvest.harvester.HarversterJobListener;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.kernel.setting.HarvesterSettingsManager;
import org.fao.geonet.repository.HarvestHistoryRepository;
import org.fao.geonet.repository.MetadataRepository;
import org.fao.geonet.repository.specification.MetadataSpecs;
import org.fao.geonet.utils.Log;
import org.fao.geonet.utils.Xml;
Expand Down Expand Up @@ -83,6 +86,8 @@ public class HarvestManagerImpl implements HarvestInfoProvider, HarvestManager {
private ServiceContext context;
private boolean readOnly;
private ConfigurableApplicationContext applicationContext;
protected MetadataRepository metadataRepository;
protected EsSearchManager searchManager;
private Map<String, AbstractHarvester> hmHarvesters = new HashMap<>();
private Map<String, AbstractHarvester> hmHarvestLookup = new HashMap<>();

Expand All @@ -108,6 +113,8 @@ public ConfigurableApplicationContext getApplicationContext() {
public void init(ServiceContext context, boolean isReadOnly) throws Exception {
this.context = context;
this.dataMan = context.getBean(DataManager.class);
this.metadataRepository = context.getBean(MetadataRepository.class);
this.searchManager = context.getBean(EsSearchManager.class);
this.settingMan = context.getBean(HarvesterSettingsManager.class);
this.translationPackBuilder = context.getBean(TranslationPackBuilder.class);

Expand Down Expand Up @@ -689,9 +696,10 @@ public synchronized OperResult clearBatch(String id) throws Exception {
long elapsedTime = System.currentTimeMillis();

String harvesterUUID = ah.getParams().getUuid();
int numberOfRecordsRemoved = metadataRepository.countByHarvestInfo_Uuid(harvesterUUID);
metadataRepository.deleteAllByHarvesterUuid(harvesterUUID);
searchManager.delete(String.format("+harvesterUuid:\"%s\"", harvesterUUID));

final Specification<Metadata> specification = (Specification<Metadata>) MetadataSpecs.hasHarvesterUuid(harvesterUUID);
int numberOfRecordsRemoved = dataMan.batchDeleteMetadataAndUpdateIndex(specification);
ah.emptyResult();
elapsedTime = (System.currentTimeMillis() - elapsedTime) / 1000;

Expand Down
22 changes: 13 additions & 9 deletions harvesters/src/main/java/org/fao/geonet/kernel/harvest/harvester/AbstractHarvester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

package org.fao.geonet.kernel.harvest.harvester;

import com.google.common.io.Files;
import jeeves.server.UserSession;
import jeeves.server.context.ServiceContext;
import org.apache.commons.lang.StringUtils;
Expand All @@ -49,11 +50,13 @@
import org.fao.geonet.kernel.datamanager.IMetadataUtils;
import org.fao.geonet.kernel.harvest.Common.OperResult;
import org.fao.geonet.kernel.harvest.Common.Status;
import org.fao.geonet.kernel.search.EsSearchManager;
import org.fao.geonet.kernel.setting.HarvesterSettingsManager;
import org.fao.geonet.kernel.setting.SettingManager;
import org.fao.geonet.kernel.setting.Settings;
import org.fao.geonet.repository.GroupRepository;
import org.fao.geonet.repository.HarvestHistoryRepository;
import org.fao.geonet.repository.MetadataRepository;
import org.fao.geonet.repository.SortUtils;
import org.fao.geonet.repository.SourceRepository;
import org.fao.geonet.repository.UserRepository;
Expand Down Expand Up @@ -81,6 +84,7 @@
import java.io.File;
import java.io.IOException;
import java.net.UnknownHostException;
import java.nio.file.FileSystem;
import java.sql.SQLException;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
Expand Down Expand Up @@ -128,6 +132,8 @@ public abstract class AbstractHarvester<T extends HarvestResult, P extends Abstr
protected DataManager dataMan;
protected IMetadataManager metadataManager;
protected IMetadataUtils metadataUtils;
protected MetadataRepository metadataRepository;
protected EsSearchManager searchManager;

protected P params;
protected T result;
Expand Down Expand Up @@ -169,9 +175,11 @@ protected void setContext(ServiceContext context) {
this.context = context;
this.dataMan = context.getBean(DataManager.class);
this.metadataUtils = context.getBean(IMetadataUtils.class);
this.metadataRepository = context.getBean(MetadataRepository.class);
this.harvesterSettingsManager = context.getBean(HarvesterSettingsManager.class);
this.settingManager = context.getBean(SettingManager.class);
this.metadataManager = context.getBean(IMetadataManager.class);
this.searchManager = context.getBean(EsSearchManager.class);
}

public void add(Element node) throws BadInputEx, SQLException {
Expand Down Expand Up @@ -271,22 +279,18 @@ public void destroy() throws Exception {
if (lock.tryLock(10, TimeUnit.SECONDS)) {

doUnschedule();

final IMetadataUtils metadataRepository = context.getBean(IMetadataUtils.class);
final IMetadataUtils metadataUtils = context.getBean(IMetadataUtils.class);
final SourceRepository sourceRepository = context.getBean(SourceRepository.class);
final Resources resources = context.getBean(Resources.class);

final Specification<? extends AbstractMetadata> ownedByHarvester = Specification.where(MetadataSpecs.hasHarvesterUuid(getParams().getUuid()));
Set<String> sources = new HashSet<>();
for (Integer metadataId : metadataRepository.findAllIdsBy(ownedByHarvester)) {
sources.add(metadataUtils.findOne(metadataId).getSourceInfo().getSourceId());
metadataManager.deleteMetadata(context, "" + metadataId);
}
List<String> sources = metadataRepository.findDistinctSourcesByHarvestInfo__uuid(getParams().getUuid());
metadataRepository.deleteAllByHarvesterUuid(getParams().getUuid());
searchManager.delete(String.format("+harvesterUuid:\"%s\"", getParams().getUuid()));

// Remove all sources related to the harvestUuid if they are not linked to any record anymore
for (String sourceUuid : sources) {
Long ownedBySource =
metadataRepository.count(Specification.where(MetadataSpecs.hasSource(sourceUuid)));
metadataUtils.count(Specification.where(MetadataSpecs.hasSource(sourceUuid)));
if (ownedBySource == 0
&& !sourceUuid.equals(params.getUuid())
&& sourceRepository.existsById(sourceUuid)) {
Expand Down
Loading