NIFI-12220 Added ability to create Controller Services from migratePr…

…operties

- Added ability to get raw property values from PropertyConfiguration instead of just effective values
- Updated TestRunner to allow for testing these migration methods
- Auto-enable newly created controller services if they are valid
- Eliminated Proxy properties in all AWS processors and instead just make use of the Proxy Configuration controller service
- Eliminated authentication properties from AWS processors and migrated all processors to using Controller Service or authentication

This closes #7874

Signed-off-by: David Handermann <exceptionfactory@apache.org>

nifi-api/src/main/java/org/apache/nifi/migration/PropertyConfiguration.java

@@ -128,10 +128,83 @@ default Optional<String> getPropertyValue(PropertyDescriptor descriptor) {
         return getPropertyValue(descriptor.getName());
     }
 
+    /**
+     * Returns an optional value representing the "raw" value of the property with the given name. The "raw" value is
+     * the value before any parameters are substituted.
+     *
+     * @param propertyName the name of the property
+     * @return an empty optional if the value is null or unset, else an Optional representing the configured value
+     */
+    Optional<String> getRawPropertyValue(String propertyName);
+
+    /**
+     * Returns an optional value representing the "raw" value of the property identified by the given descriptor. The "raw" value is
+     * the value before any parameters are substituted.
+     *
+     * @param descriptor the descriptor that identifies the property
+     * @return an empty optional if the value is null or unset, else an Optional representing the configured value
+     */
+    default Optional<String> getRawPropertyValue(PropertyDescriptor descriptor) {
+        return getRawPropertyValue(descriptor.getName());
+    }
+
     /**
      * Returns a map containing all of the configured properties
      * @return a Map containing the names and values of all configured properties
      */
     Map<String, String> getProperties();
 
+    /**
+     * Returns a map containing all of the raw property values
+     *
+     * @return a Map containing the names and values of all configured properties
+     */
+    Map<String, String> getRawProperties();
+
+    /**
+     * <p>
+     * Creates a new Controller Service of the given type and configures it with the given property values. Note that if a Controller Service
+     * already exists within the same scope and with the same implementation and configuration, a new service may not be created and instead
+     * the existing service may be used.
+     * </p>
+     *
+     * <p>
+     * This allows for properties that were previously defined in the extension to be moved to a Controller Service. For example,
+     * consider a Processor that has "Username" and "Password" properties. In the next version of the Processor, we want to support
+     * multiple types of authentication, and we delegate the authentication to a Controller Service. Consider that the Controller Service
+     * implementation we wish to use has a classname of {@code org.apache.nifi.services.authentication.UsernamePassword}. We might then
+     * use this method as such:
+     * </p>
+     *
+     * <pre><code>
+     *     // Create a new Controller Service of type org.apache.nifi.services.authentication.UsernamePassword whose Username and Password
+     *     // properties match those currently configured for this Processor.
+     *     final Map&lt;String, String&gt; serviceProperties = Map.of("Username", propertyConfiguration.getRawPropertyValue("Username"),
+     *          "Password", propertyConfiguration.getRawPropertyValue("Password"));
+     *     final String serviceId = propertyConfiguration.createControllerService("org.apache.nifi.services.authentication.UsernamePassword", serviceProperties);
+     *
+     *     // Set our Authentication Service property to point to this new service.
+     *     propertyConfiguration.setProperty(AUTHENTICATION_SERVICE, serviceId);
+     *
+     *     // Remove the Username and Password properties from this Processor, since we are now going to use then Authentication Service.
+     *     propertyConfiguration.removeProperty("Username");
+     *     propertyConfiguration.removeProperty("Password");
+     * </code></pre>
+     *
+     * <p>
+     * Note the use of {@link #getRawPropertyValue(String)} here instead of {@link #getPropertyValue(String)}. Because we want to set
+     * the new Controller Service's value to the same value as is currently configured for the Processor's "Username" and "Password" properties,
+     * we use {@link #getRawPropertyValue(String)}. This ensures that if the Processor is configured using Parameters, those Parameter
+     * references are still held by the Controller Service.
+     * </p>
+     *
+     * <p>
+     * Also note that this method expects the classname of the implementation, not the classname of the interface.
+     * </p>
+     *
+     * @param implementationClassName the fully qualified classname of the Controller Service implementation
+     * @param serviceProperties       the property values to configure the newly created Controller Service with
+     * @return an identifier for the Controller Service
+     */
+    String createControllerService(String implementationClassName, Map<String, String> serviceProperties);
 }

nifi-mock/src/main/java/org/apache/nifi/util/MockPropertyConfiguration.java

@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.util;
+
+import org.apache.nifi.migration.PropertyConfiguration;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+
+public class MockPropertyConfiguration implements PropertyConfiguration {
+    private final Map<String, String> propertyRenames = new HashMap<>();
+    private final Set<String> propertiesRemoved = new HashSet<>();
+    private final Set<String> propertiesUpdated = new HashSet<>();
+    private final Map<String, String> rawProperties;
+    private final Set<CreatedControllerService> createdControllerServices = new HashSet<>();
+
+
+    public MockPropertyConfiguration(final Map<String, String> propertyValues) {
+        this.rawProperties = new HashMap<>(propertyValues);
+    }
+
+    public PropertyMigrationResult toPropertyMigrationResult() {
+        return new PropertyMigrationResult() {
+
+            @Override
+            public Set<String> getPropertiesRemoved() {
+                return Collections.unmodifiableSet(propertiesRemoved);
+            }
+
+            @Override
+            public Map<String, String> getPropertiesRenamed() {
+                return Collections.unmodifiableMap(propertyRenames);
+            }
+
+            @Override
+            public Set<CreatedControllerService> getCreatedControllerServices() {
+                return Collections.unmodifiableSet(createdControllerServices);
+            }
+
+            @Override
+            public Set<String> getPropertiesUpdated() {
+                return Collections.unmodifiableSet(propertiesUpdated);
+            }
+        };
+    }
+
+    @Override
+    public boolean renameProperty(final String propertyName, final String newName) {
+        propertyRenames.put(propertyName, newName);
+
+        final boolean hasProperty = hasProperty(propertyName);
+        if (!hasProperty) {
+            return false;
+        }
+
+        final String value = rawProperties.remove(propertyName);
+        rawProperties.put(newName, value);
+        return true;
+    }
+
+    @Override
+    public boolean removeProperty(final String propertyName) {
+        propertiesRemoved.add(propertyName);
+
+        if (!hasProperty(propertyName)) {
+            return false;
+        }
+
+        rawProperties.remove(propertyName);
+        return true;
+    }
+
+    @Override
+    public boolean hasProperty(final String propertyName) {
+        return rawProperties.containsKey(propertyName);
+    }
+
+    @Override
+    public boolean isPropertySet(final String propertyName) {
+        return rawProperties.get(propertyName) != null;
+    }
+
+    @Override
+    public void setProperty(final String propertyName, final String propertyValue) {
+        propertiesUpdated.add(propertyName);
+        rawProperties.put(propertyName, propertyValue);
+    }
+
+    @Override
+    public Optional<String> getPropertyValue(final String propertyName) {
+        return getRawPropertyValue(propertyName);
+    }
+
+    @Override
+    public Optional<String> getRawPropertyValue(final String propertyName) {
+        return Optional.ofNullable(rawProperties.get(propertyName));
+    }
+
+    @Override
+    public Map<String, String> getProperties() {
+        return getRawProperties();
+    }
+
+    @Override
+    public Map<String, String> getRawProperties() {
+        return Collections.unmodifiableMap(rawProperties);
+    }
+
+    @Override
+    public String createControllerService(final String implementationClassName, final Map<String, String> serviceProperties) {
+        final String serviceId = UUID.randomUUID().toString();
+        createdControllerServices.add(new CreatedControllerService(serviceId, implementationClassName, serviceProperties));
+        return serviceId;
+    }
+
+    public record CreatedControllerService(String id, String implementationClassName, Map<String, String> serviceProperties) {
+    }
+}

nifi-mock/src/main/java/org/apache/nifi/util/PropertyMigrationResult.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.util;
+
+import org.apache.nifi.components.PropertyDescriptor;
+
+import java.util.Map;
+import java.util.Set;
+
+public interface PropertyMigrationResult {
+
+    /**
+     * @return a set containing the names of all properties that were removed
+     */
+    Set<String> getPropertiesRemoved();
+
+    /**
+     * @return a mapping of previous property names to the new names of those properties
+     */
+    Map<String, String> getPropertiesRenamed();
+
+    /**
+     * @return a set of all controller services that were added
+     */
+    Set<MockPropertyConfiguration.CreatedControllerService> getCreatedControllerServices();
+
+    /**
+     * @return a set of all properties whose values were updated via calls to {@link org.apache.nifi.migration.PropertyConfiguration#setProperty(String, String)} or
+     * {@link org.apache.nifi.migration.PropertyConfiguration#setProperty(PropertyDescriptor, String)}.
+     */
+    Set<String> getPropertiesUpdated();
+}

nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java

@@ -1066,4 +1066,49 @@ public void assertProvenanceEvent(final ProvenanceEventType eventType) {
                         .collect(toSet());
         assertEquals(expectedEventTypes, actualEventTypes);
     }
+
+    @Override
+    public PropertyMigrationResult migrateProperties() {
+        final MockPropertyConfiguration mockPropertyConfiguration = new MockPropertyConfiguration(getProcessContext().getAllProperties());
+        getProcessor().migrateProperties(mockPropertyConfiguration);
+
+        final PropertyMigrationResult migrationResult = mockPropertyConfiguration.toPropertyMigrationResult();
+        final Set<MockPropertyConfiguration.CreatedControllerService> services = migrationResult.getCreatedControllerServices();
+
+        RuntimeException serviceCreationException = null;
+        for (final MockPropertyConfiguration.CreatedControllerService service : services) {
+            final ControllerService serviceImpl;
+            try {
+                final Class<?> clazz = Class.forName(service.implementationClassName());
+                final Object newInstance = clazz.getDeclaredConstructor().newInstance();
+                if (!(newInstance instanceof ControllerService)) {
+                    throw new RuntimeException(clazz + " is not a Controller Service");
+                }
+
+                serviceImpl = (ControllerService) newInstance;
+                addControllerService(service.id(), serviceImpl, service.serviceProperties());
+            } catch (final Exception e) {
+                if (serviceCreationException == null) {
+                    if (e instanceof RuntimeException) {
+                        serviceCreationException = (RuntimeException) e;
+                    } else {
+                        serviceCreationException = new RuntimeException(e);
+                    }
+                } else {
+                    serviceCreationException.addSuppressed(e);
+                }
+            }
+        }
+
+        if (serviceCreationException != null) {
+            throw serviceCreationException;
+        }
+
+        final Map<String, String> updatedProperties = mockPropertyConfiguration.getRawProperties();
+        final MockProcessContext processContext = getProcessContext();
+        processContext.clearProperties();
+        updatedProperties.forEach(processContext::setProperty);
+
+        return migrationResult;
+    }
 }

nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java

@@ -22,6 +22,7 @@
 import org.apache.nifi.controller.ControllerService;
 import org.apache.nifi.controller.queue.QueueSize;
 import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.migration.PropertyConfiguration;
 import org.apache.nifi.processor.ProcessContext;
 import org.apache.nifi.processor.ProcessSession;
 import org.apache.nifi.processor.ProcessSessionFactory;
@@ -1064,4 +1065,13 @@ void assertAttributes(
      * @param eventType Provenance event type
      */
      void assertProvenanceEvent(ProvenanceEventType eventType);
+
+    /**
+     * Causes the TestRunner to call the Processor's {@link Processor#migrateProperties(PropertyConfiguration)} method. The effects that are
+     * caused by calling the method are applied, as they would be in a running NiFi instance. Unlike in a running NiFi instance, though, the
+     * operations that were performed are captured so that they can be examined and assertions made about the migration that occurred.
+     *
+     * @return the results of migrating properties
+     */
+    PropertyMigrationResult migrateProperties();
 }