Merge pull request #299 from ppalaga/i298

Fix #298 Add removeOrphanLicenseFiles param to AbstractDownloadLicens…

src/it/ISSUE-197/invoker.properties

@@ -1 +1 @@
-invoker.goals = license:download-licenses
+invoker.goals = license:download-licenses -e

src/it/download-licenses-configured/licenses-delete-orphans.expected.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<licenseSummary>
+  <dependencies>
+    <dependency>
+      <groupId>aopalliance</groupId>
+      <artifactId>aopalliance</artifactId>
+      <version>1.0</version>
+      <licenses>
+        <license>
+          <name>Public Domain</name>
+        </license>
+      </licenses>
+    </dependency>
+    <dependency>
+      <groupId>asm</groupId>
+      <artifactId>asm</artifactId>
+      <version>3.1</version>
+      <licenses>
+        <license>
+          <name>BSD 3-Clause ASM</name>
+          <url>https://gitlab.ow2.org/asm/asm/raw/ASM_3_1_MVN/LICENSE.txt</url>
+          <file>bsd-3-clause-asm-license.txt</file>
+        </license>
+      </licenses>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>1.0</version>
+      <licenses>
+        <license>
+          <name>Apache License 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+          <file>apache-license-2.0-license-2.0.txt</file>
+        </license>
+      </licenses>
+    </dependency>
+  </dependencies>
+</licenseSummary>

src/it/download-licenses-configured/pom.xml

@@ -108,6 +108,25 @@
               </licenseUrlFileNameSanitizers>
             </configuration>
           </execution>
+          <execution>
+            <id>delete-orphans</id>
+            <phase>validate</phase>
+            <goals>
+              <goal>download-licenses</goal>
+            </goals>
+            <configuration>
+              <licensesConfigFile>${basedir}/src/license/licenses-config-delete-orphans.xml</licensesConfigFile>
+              <licensesOutputDirectory>${project.build.directory}/delete-orphans/licenses</licensesOutputDirectory>
+              <licensesOutputFile>${project.build.directory}/delete-orphans/licenses.xml</licensesOutputFile>
+              <sortByGroupIdAndArtifactId>true</sortByGroupIdAndArtifactId>
+              <licenseUrlFileNameSanitizers>
+                <licenseUrlFileNameSanitizer>
+                  <regexp>[\s-_]+</regexp>
+                  <replacement>-</replacement>
+                </licenseUrlFileNameSanitizer>
+              </licenseUrlFileNameSanitizers>
+            </configuration>
+          </execution>
           <execution>
             <id>insert-versions</id>
             <phase>validate</phase>

src/it/download-licenses-configured/postbuild.groovy

@@ -80,6 +80,25 @@ return {
     assert Files.exists(bsdAsm)
     assert bsdAsm.text.contains('Fake content')
 
+    final Path expectedLicensesXml = basePath.resolve('licenses-'+ id +'.expected.xml')
+    final Path licensesXml = outputBase.resolve('licenses.xml')
+    assert expectedLicensesXml.text.equals(licensesXml.text)
+    return true
+}() && {
+    final String id = 'delete-orphans'
+    final Path outputBase = basePath.resolve('target/' + id)
+
+    final Path asl2 = outputBase.resolve('licenses/apache-license-2.0-license-2.0.txt')
+    assert Files.exists(asl2)
+    assert asl2.text.contains('Fake content')
+
+    final Path bsdAsm = outputBase.resolve('licenses/bsd-3-clause-asm-license.txt')
+    assert Files.exists(bsdAsm)
+    assert bsdAsm.text.contains('Fake content')
+
+    final Path fooBar = outputBase.resolve('licenses/foo-bar-license.txt')
+    assert !Files.exists(fooBar)
+
     final Path expectedLicensesXml = basePath.resolve('licenses-'+ id +'.expected.xml')
     final Path licensesXml = outputBase.resolve('licenses.xml')
     assert expectedLicensesXml.text.equals(licensesXml.text)

src/it/download-licenses-configured/src/license/licenses-config-delete-orphans.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<licenseSummary>
+  <dependencies>
+    <dependency>
+      <groupId>\Qcommons-logging</groupId>
+      <artifactId>\Qcommons-logging</artifactId>
+      <matchLicenses>
+        <!-- match empty license list -->
+      </matchLicenses>
+      <licenses>
+        <license>
+          <name>Apache License 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+        </license>
+      </licenses>
+    </dependency>
+    <dependency>
+      <groupId>\Qaopalliance\E</groupId>
+      <artifactId>\Qaopalliance\E</artifactId>
+      <matchLicenses>
+        <license>
+          <name>\QPublic Domain\E</name>
+        </license>
+      </matchLicenses>
+      <licenses approved="true" />
+    </dependency>
+    <dependency>
+      <groupId>\Qasm\E</groupId>
+      <artifactId>\Qasm\E</artifactId>
+      <matchLicenses>
+        <!-- match empty license list -->
+      </matchLicenses>
+      <licenses>
+        <license>
+          <name>BSD 3-Clause ASM</name>
+          <url>https://gitlab.ow2.org/asm/asm/raw/ASM_3_1_MVN/LICENSE.txt</url>
+        </license>
+      </licenses>
+    </dependency>
+  </dependencies>
+</licenseSummary>

src/it/download-licenses-configured/target-initial/delete-orphans/licenses.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<licenseSummary>
+  <dependencies>
+    <dependency>
+      <groupId>aopalliance</groupId>
+      <artifactId>aopalliance</artifactId>
+      <version>1.0</version>
+      <licenses>
+        <license>
+          <name>Public Domain</name>
+        </license>
+      </licenses>
+    </dependency>
+    <dependency>
+      <groupId>asm</groupId>
+      <artifactId>asm</artifactId>
+      <version>3.1</version>
+      <licenses>
+        <license>
+          <name>BSD 3-Clause ASM</name>
+          <url>https://gitlab.ow2.org/asm/asm/raw/ASM_3_1_MVN/LICENSE.txt</url>
+          <file>bsd-3-clause-asm-license.txt</file>
+        </license>
+      </licenses>
+    </dependency>
+    <dependency>
+      <groupId>foo</groupId>
+      <artifactId>bar</artifactId>
+      <version>1.2.3</version>
+      <licenses>
+        <license>
+          <name>The Foo Bar License v42</name>
+          <url>https://foo.bar/LICENSE.txt</url>
+          <file>foo-bar-license.txt</file>
+        </license>
+      </licenses>
+    </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>1.0</version>
+      <licenses>
+        <license>
+          <name>Apache License 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+          <file>apache-license-2.0-license-2.0.txt</file>
+        </license>
+      </licenses>
+    </dependency>
+  </dependencies>
+</licenseSummary>

src/it/download-licenses-configured/target-initial/delete-orphans/licenses/apache-license-2.0-license-2.0.txt

@@ -0,0 +1 @@
+Fake content that should not get overwritten without forceDownload=true

src/it/download-licenses-configured/target-initial/delete-orphans/licenses/bsd-3-clause-asm-license.txt

@@ -0,0 +1 @@
+Fake content that should not get overwritten without forceDownload=true

src/it/download-licenses-configured/target-initial/delete-orphans/licenses/foo-bar-license.txt

@@ -0,0 +1 @@
+Should be deleted due to removeOrphanLicenseFiles=true

src/main/java/org/codehaus/mojo/license/AbstractDownloadLicensesMojo.java

@@ -56,6 +56,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Locale;
@@ -202,11 +203,29 @@ public abstract class AbstractDownloadLicensesMojo
      * {@code cleanLicensesOutputDirectory = true} is not implied by {@link #forceDownload} because users may have
      * other files there in {@link #licensesOutputDirectory} that were not downloaded by the plugin.
      *
+     * @see #removeOrphanLicenseFiles
      * @since 1.18
      */
     @Parameter( property = "license.cleanLicensesOutputDirectory", defaultValue = "false" )
     private boolean cleanLicensesOutputDirectory;
 
+    /**
+     * If {@code true} the files referenced from {@link AbstractLicensesXmlMojo#licensesOutputFile} before executing
+     * the mojo but not referenced from {@link AbstractLicensesXmlMojo#licensesOutputFile} after executing
+     * the mojo will be deleted; otherwise neither before:after diffing nor any file deletions will happen.
+     * <p>
+     * Compared to {@link #cleanLicensesOutputDirectory} that removes all files from {@link #licensesOutputDirectory}
+     * before downloading all licenses anew, the {@link #removeOrphanLicenseFiles} removes only files that
+     * are certainly not needed anymore, e.g. due to a removal of a dependency. {@link #removeOrphanLicenseFiles} thus
+     * allows to avoid downloading the license files of dependencies that were downloaded in the past and are still
+     * available in {@link #licensesOutputDirectory}.
+     *
+     * @see #cleanLicensesOutputDirectory
+     * @since 1.19
+     */
+    @Parameter( property = "license.removeOrphanLicenseFiles", defaultValue = "true" )
+    private boolean removeOrphanLicenseFiles;
+
     /**
      * A file containing dependencies whose licenses could not be downloaded for some reason. The format is similar to
      * {@link #licensesOutputFile} but the entries in {@link #licensesErrorsFile} have {@code <downloaderMessage>}
@@ -581,6 +600,7 @@ public abstract class AbstractDownloadLicensesMojo
     private int downloadErrorCount = 0;
 
     private ArtifactFilters artifactFilters;
+    private final Set<String> orphanFileNames = new HashSet<>();
 
     protected abstract boolean isSkip();
 
@@ -636,15 +656,19 @@ public void execute()
                     for ( ProjectLicense lic : dep.getLicenses() )
                     {
                         final String fileName = lic.getFile();
-                        final String url = lic.getUrl();
-                        if ( fileName != null && url != null )
+                        if ( fileName != null )
                         {
-                            final File file = new File( licensesOutputDirectory, fileName );
-                            if ( file.exists() )
+                            orphanFileNames.add( fileName );
+                            final String url = lic.getUrl();
+                            if ( url != null )
                             {
-                                final LicenseDownloadResult entry =
-                                    LicenseDownloadResult.success( file, FileUtil.sha1( file.toPath() ), false );
-                                cache.put( url, entry );
+                                final File file = new File( licensesOutputDirectory, fileName );
+                                if ( file.exists() )
+                                {
+                                    final LicenseDownloadResult entry =
+                                        LicenseDownloadResult.success( file, FileUtil.sha1( file.toPath() ), false );
+                                    cache.put( url, entry );
+                                }
                             }
                         }
                     }
@@ -708,6 +732,8 @@ public void execute()
             {
                 writeLicenseSummary( depProjectLicensesWithErrors, licensesErrorsFile, writeVersions );
             }
+
+            removeOrphanFiles( depProjectLicenses );
         }
         catch ( Exception e )
         {
@@ -736,6 +762,30 @@ public void execute()
         }
     }
 
+    private void removeOrphanFiles( List<ProjectLicenseInfo> deps )
+    {
+        if ( removeOrphanLicenseFiles )
+        {
+            for ( ProjectLicenseInfo dep : deps )
+            {
+                for ( ProjectLicense lic : dep.getLicenses() )
+                {
+                    orphanFileNames.remove( lic.getFile() );
+                }
+            }
+
+            for ( String fileName : orphanFileNames )
+            {
+                final File file = new File( licensesOutputDirectory, fileName );
+                if ( file.exists() )
+                {
+                    getLog().info( "Removing orphan license file \"" + file + "\"" );
+                    file.delete();
+                }
+            }
+        }
+    }
+
     /**
      * Removes from the given {@code depProjectLicenses} those elements which have non-empty
      * {@link ProjectLicenseInfo#getDownloaderMessages()} and adds those to the resulting {@link List}.