Merge branch 'versions' into upgrade

src/main/java/gov/nasa/pds/registry/mgr/RegistryManagerCli.java

@@ -17,6 +17,7 @@
 import gov.nasa.pds.registry.mgr.cmd.data.DeleteDataCmd;
 import gov.nasa.pds.registry.mgr.cmd.data.ExportFileCmd;
 import gov.nasa.pds.registry.mgr.cmd.data.SetArchiveStatusCmd;
+import gov.nasa.pds.registry.mgr.cmd.data.UpdateAltIdsCmd;
 import gov.nasa.pds.registry.mgr.cmd.dd.DeleteDDCmd;
 import gov.nasa.pds.registry.mgr.cmd.dd.ExportDDCmd;
 import gov.nasa.pds.registry.mgr.cmd.dd.ListDDCmd;
@@ -65,6 +66,7 @@ public static void printHelp()
         System.out.println("  delete-data          Delete data from registry index");
         System.out.println("  export-file          Export a file from blob storage");
         System.out.println("  set-archive-status   Set product archive status");
+        System.out.println("  update-alt-ids       Update alternate IDs");
 
         System.out.println();
         System.out.println("Registry:");
@@ -244,9 +246,9 @@ private void initCommands()
 
         // Data
         commands.put("delete-data", new DeleteDataCmd());
-        //commands.put("export-data", new ExportDataCmd());
         commands.put("export-file", new ExportFileCmd());
         commands.put("set-archive-status", new SetArchiveStatusCmd());
+        commands.put("update-alt-ids", new UpdateAltIdsCmd());
     }
 
 

src/main/java/gov/nasa/pds/registry/mgr/cmd/data/UpdateAltIdsCmd.java

@@ -0,0 +1,183 @@
+package gov.nasa.pds.registry.mgr.cmd.data;
+
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.ResponseException;
+
+import gov.nasa.pds.registry.common.cfg.RegistryCfg;
+import gov.nasa.pds.registry.common.es.client.EsUtils;
+import gov.nasa.pds.registry.common.util.CloseUtils;
+import gov.nasa.pds.registry.mgr.Constants;
+import gov.nasa.pds.registry.mgr.cmd.CliCommand;
+import gov.nasa.pds.registry.mgr.dao.RegistryDao;
+import gov.nasa.pds.registry.mgr.dao.RegistryManager;
+
+
+/**
+ * A CLI command to update product's alternate IDs in Elasticsearch.
+ * 
+ * @author karpenko
+ */
+public class UpdateAltIdsCmd implements CliCommand
+{
+    private Logger log;
+
+    /**
+     * Constructor
+     */
+    public UpdateAltIdsCmd()
+    {
+        log = LogManager.getLogger(this.getClass());
+    }
+
+
+    @Override
+    public void run(CommandLine cmdLine) throws Exception
+    {
+        if(cmdLine.hasOption("help"))
+        {
+            printHelp();
+            return;
+        }
+
+        RegistryCfg cfg = new RegistryCfg();
+        cfg.url = cmdLine.getOptionValue("es", "http://localhost:9200");
+        cfg.indexName = cmdLine.getOptionValue("index", Constants.DEFAULT_REGISTRY_INDEX);
+        cfg.authFile = cmdLine.getOptionValue("auth");
+
+        String filePath = cmdLine.getOptionValue("file");
+        if(filePath == null) throw new Exception("Missing required parameter '-file'");
+        File file = new File(filePath);
+        if(!file.exists()) throw new Exception("Input file doesn't exist: " + file.getAbsolutePath());
+
+        try
+        {
+            RegistryManager.init(cfg);
+            updateIds(file);
+        }
+        catch(ResponseException ex)
+        {
+            throw new Exception(EsUtils.extractErrorMessage(ex));
+        }
+        finally
+        {
+            RegistryManager.destroy();
+        }
+    }
+
+
+    private void updateIds(File file) throws Exception
+    {
+        BufferedReader rd = null;
+
+        try
+        {
+            rd = new BufferedReader(new FileReader(file));
+
+            String line;
+            int lineNum = 0;
+
+            while((line = rd.readLine()) != null)
+            {
+                lineNum++;
+                line = line.trim();
+
+                String[] ids = StringUtils.split(line, ",;\t ");
+                if(ids == null || ids.length != 2)
+                {
+                    log.warn("Line " + lineNum + " has invalid value: [" + line + "]");
+                    continue;
+                }
+
+                List<String> newIds = new ArrayList<String>(4);
+
+                // First LIDVID value (old)
+                int idx = ids[0].indexOf("::");
+                if(idx < 1)
+                {
+                    log.warn("Line " + lineNum + " has invalid LIDVID: [" + ids[0] + "]");
+                    continue;
+                }
+
+                // Add LIDVID
+                newIds.add(ids[0]);
+                // Add LID
+                newIds.add(ids[0].substring(0, idx));
+
+                // Second LIDVID value (new)
+                idx = ids[1].indexOf("::");
+                if(idx < 1)
+                {
+                    log.warn("Line " + lineNum + " has invalid LIDVID: [" + ids[1] + "]");
+                    continue;
+                }
+
+                // Add LIDVID
+                newIds.add(ids[1]);
+                // Add LID
+                newIds.add(ids[1].substring(0, idx));
+
+                Map<String, List<String>> idMap = new TreeMap<>();
+                idMap.put(ids[0], newIds);
+                idMap.put(ids[1], newIds);
+
+                log.info("Updating " + ids[0] + " -> " + ids[1]);
+                updateIds(idMap);
+            }
+        }
+        finally
+        {
+            CloseUtils.close(rd);
+        }
+    }
+
+
+    private void updateIds(Map<String, List<String>> newIds) throws Exception
+    {
+        RegistryDao dao = RegistryManager.getInstance().getRegistryDao();
+
+        Map<String, Set<String>> existingIds = dao.getAlternateIds(newIds.keySet());
+        if(existingIds == null || existingIds.isEmpty()) return;
+
+        for(Map.Entry<String, Set<String>> entry: existingIds.entrySet())            
+        {
+            List<String> additionalIds = newIds.get(entry.getKey());
+            if(additionalIds != null)
+            {
+                entry.getValue().addAll(additionalIds);
+            }
+        }
+
+        dao.updateAlternateIds(existingIds);
+    }
+
+
+    public void printHelp()
+    {
+        System.out.println("Usage: registry-manager update-alt-ids <options>");
+
+        System.out.println();
+        System.out.println("Update product's alternate IDs");
+        System.out.println();
+        System.out.println("Required parameters:");
+        System.out.println("  -file <path>      CSV file with the list of IDs");
+        System.out.println("Optional parameters:");
+        System.out.println("  -auth <file>      Authentication config file");
+        System.out.println("  -es <url>         Elasticsearch URL. Default is http://localhost:9200");
+        System.out.println("  -index <name>     Elasticsearch index name. Default is 'registry'");
+        System.out.println();
+    }
+
+}

src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryDao.java

@@ -0,0 +1,125 @@
+package gov.nasa.pds.registry.mgr.dao;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.RestClient;
+
+import gov.nasa.pds.registry.common.es.client.SearchResponseParser;
+import gov.nasa.pds.registry.common.es.dao.BulkResponseParser;
+import gov.nasa.pds.registry.common.util.CloseUtils;
+import gov.nasa.pds.registry.mgr.dao.resp.GetAltIdsParser;
+
+/**
+ * Data access object
+ * @author karpenko
+ */
+public class RegistryDao
+{
+    private Logger log;
+
+    private RestClient client;
+    private String indexName;
+
+    private boolean pretty = false;
+
+    /**
+     * Constructor
+     * @param client Elasticsearch client
+     * @param indexName Elasticsearch index
+     */
+    public RegistryDao(RestClient client, String indexName)
+    {
+        log = LogManager.getLogger(this.getClass());
+
+        this.client = client;
+        this.indexName = indexName;
+    }
+
+
+    /**
+     * Generate pretty JSONs for debugging
+     * @param b boolean flag
+     */
+    public void setPretty(boolean b)
+    {
+        this.pretty = b;
+    }
+
+
+    /**
+     * Get product's alternative IDs by primary key 
+     * @param ids primary keys (usually LIDVIDs)
+     * @return ID map: key = product primary key (usually LIDVID), value = set of alternate IDs 
+     * @throws Exception an exception
+     */
+    public Map<String, Set<String>> getAlternateIds(Collection<String> ids) throws Exception
+    {
+        if(ids == null || ids.isEmpty()) return null;
+
+        RegistryRequestBuilder bld = new RegistryRequestBuilder();
+        String jsonReq = bld.createGetAlternateIdsRequest(ids);
+
+        String reqUrl = "/" + indexName + "/_search";
+        if(pretty) reqUrl += "?pretty";
+
+        Request req = new Request("GET", reqUrl);
+        req.setJsonEntity(jsonReq);
+        Response resp = client.performRequest(req);
+
+        //DebugUtils.dumpResponseBody(resp);
+
+        GetAltIdsParser cb = new GetAltIdsParser();
+        SearchResponseParser parser = new SearchResponseParser();
+        parser.parseResponse(resp, cb);
+
+        return cb.getIdMap();
+    }
+
+
+    /**
+     * Update alternate IDs by primary keys
+     * @param newIds ID map: key = product primary key (usually LIDVID), 
+     * value = additional alternate IDs to be added to existing alternate IDs.
+     * @throws Exception an exception
+     */
+    public void updateAlternateIds(Map<String, Set<String>> newIds) throws Exception
+    {
+        if(newIds == null || newIds.isEmpty()) return;
+
+        RegistryRequestBuilder bld = new RegistryRequestBuilder();
+        String json = bld.createUpdateAltIdsRequest(newIds);
+        log.debug("Request:\n" + json);
+
+        String reqUrl = "/" + indexName + "/_bulk"; //?refresh=wait_for";
+        Request req = new Request("POST", reqUrl);
+        req.setJsonEntity(json);
+
+        Response resp = client.performRequest(req);
+
+        // Check for Elasticsearch errors.
+        InputStream is = null;
+        InputStreamReader rd = null;
+        try
+        {
+            is = resp.getEntity().getContent();
+            rd = new InputStreamReader(is);
+
+            BulkResponseParser parser = new BulkResponseParser();
+            parser.parse(rd);
+        }
+        finally
+        {
+            CloseUtils.close(rd);
+            CloseUtils.close(is);
+        }
+    }
+
+}

src/main/java/gov/nasa/pds/registry/mgr/dao/RegistryManager.java

@@ -23,6 +23,7 @@ public class RegistryManager
     private RestClient esClient;
     private SchemaDao schemaDao;
     private DataDictionaryDao dataDictionaryDao;
+    private RegistryDao registryDao;
 
 
     /**
@@ -47,7 +48,8 @@ private RegistryManager(RegistryCfg cfg) throws Exception
         log.info("Registry index: " + indexName);
 
         schemaDao = new SchemaDao(esClient, indexName);
-        dataDictionaryDao = new DataDictionaryDao(esClient, indexName);                
+        dataDictionaryDao = new DataDictionaryDao(esClient, indexName);
+        registryDao = new RegistryDao(esClient, indexName);
     }
 
 
@@ -103,4 +105,14 @@ public DataDictionaryDao getDataDictionaryDao()
         return dataDictionaryDao;
     }
 
+
+    /**
+     * Get registry DAO object.
+     * @return Registry DAO
+     */
+    public RegistryDao getRegistryDao()
+    {
+        return registryDao;
+    }
+
 }