Add config keys for controlling public/private template secondary storage replica counts

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java

@@ -67,6 +67,8 @@ public TemplateInfo getTemplate() {
 
     void handleTemplateSync(DataStore store);
 
+    void enforceSecStorageCopyLimit(long templateId, long zoneId);
+
     void downloadBootstrapSysTemplate(DataStore store);
 
     void addSystemVMTemplatesToSecondary(DataStore store);

engine/components-api/src/main/java/com/cloud/template/TemplateManager.java

@@ -45,6 +45,8 @@
 public interface TemplateManager {
     static final String AllowPublicUserTemplatesCK = "allow.public.user.templates";
     static final String TemplatePreloaderPoolSizeCK = "template.preloader.pool.size";
+    static final String PublicTemplateSecStorageCopyCK = "secstorage.public.template.copy.max";
+    static final String PrivateTemplateSecStorageCopyCK = "secstorage.private.template.copy.max";
 
     static final ConfigKey<Boolean> AllowPublicUserTemplates = new ConfigKey<Boolean>("Advanced", Boolean.class, AllowPublicUserTemplatesCK, "true",
         "If false, users will not be able to create public Templates.", true, ConfigKey.Scope.Account);
@@ -64,6 +66,18 @@ public interface TemplateManager {
             true,
             ConfigKey.Scope.Global);
 
+    ConfigKey<Integer> PublicTemplateSecStorageCopy = new ConfigKey<Integer>("Advanced", Integer.class,
+            PublicTemplateSecStorageCopyCK, "0",
+            "Maximum number of secondary storage pools to which a public template is copied. " +
+            "0 means copy to all secondary storage pools (default behavior).",
+            true, ConfigKey.Scope.Zone);
+
+    ConfigKey<Integer> PrivateTemplateSecStorageCopy = new ConfigKey<Integer>("Advanced", Integer.class,
+            PrivateTemplateSecStorageCopyCK, "1",
+            "Maximum number of secondary storage pools to which a private template is copied. " +
+            "Default is 1 to preserve existing behavior.",
+            true, ConfigKey.Scope.Zone);
+
     static final String VMWARE_TOOLS_ISO = "vmware-tools.iso";
     static final String XS_TOOLS_ISO = "xs-tools.iso";
 
@@ -138,6 +152,12 @@ public interface TemplateManager {
 
     List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId);
 
+    /**
+     * Max number of secondary storage copies for the template in this zone; {@code 0} means no limit.
+     * SYSTEM/ROUTING/BUILTIN templates are always exempt (returns {@code 0}).
+     */
+    int getSecStorageCopyLimit(VMTemplateVO template, long zoneId);
+
     TemplateInfo prepareIso(long isoId, long dcId, Long hostId, Long poolId);
 
 

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -295,6 +295,93 @@ public void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId) {
         }
     }
 
+    private boolean hasReachedSecStorageCopyLimit(VMTemplateVO template, long zoneId) {
+        int copyLimit = _tmpltMgr.getSecStorageCopyLimit(template, zoneId);
+        if (copyLimit <= 0) {
+            return false;
+        }
+        List<DataStore> stores = _storeMgr.getImageStoresByScope(new ZoneScope(zoneId));
+        if (stores == null || stores.isEmpty()) {
+            return false;
+        }
+        int count = 0;
+        for (DataStore ds : stores) {
+            List<TemplateDataStoreVO> rows = _vmTemplateStoreDao.listByTemplateStore(template.getId(), ds.getId());
+            if (rows == null) {
+                continue;
+            }
+            for (TemplateDataStoreVO row : rows) {
+                State st = row.getState();
+                Status ds_state = row.getDownloadState();
+                if (st != State.Failed && st != State.Destroyed
+                        && ds_state != Status.ABANDONED && ds_state != Status.DOWNLOAD_ERROR) {
+                    count++;
+                    break;
+                }
+            }
+        }
+        logger.debug("Template [{}] secstorage copy check in zone [{}]: count={}, limit={}",
+                template.getUniqueName(), zoneId, count, copyLimit);
+        return count >= copyLimit;
+    }
+
+    @Override
+    public void enforceSecStorageCopyLimit(long templateId, long zoneId) {
+        VMTemplateVO template = _templateDao.findById(templateId);
+        if (template == null) {
+            return;
+        }
+        int copyLimit = _tmpltMgr.getSecStorageCopyLimit(template, zoneId);
+        if (copyLimit <= 0) {
+            return;
+        }
+        if (_tmpltMgr.verifyHeuristicRulesForZone(template, zoneId) != null) {
+            return;
+        }
+        GlobalLock lock = GlobalLock.getInternLock("template.copy.limit." + templateId + "." + zoneId);
+        try {
+            if (!lock.lock(30)) {
+                logger.warn("Could not acquire lock to enforce secondary storage copy limit for template [{}] in zone [{}].",
+                        template.getUniqueName(), zoneId);
+                return;
+            }
+            List<DataStore> stores = _storeMgr.getImageStoresByScope(new ZoneScope(zoneId));
+            if (stores == null) {
+                return;
+            }
+            List<TemplateDataStoreVO> removable = new ArrayList<>();
+            for (DataStore ds : stores) {
+                TemplateDataStoreVO ref = _vmTemplateStoreDao.findByStoreTemplate(ds.getId(), templateId);
+                if (ref != null
+                        && ref.getState() == State.Ready
+                        && ref.getDownloadState() == Status.DOWNLOADED
+                        && (ref.getRefCnt() == null || ref.getRefCnt() == 0)) {
+                    removable.add(ref);
+                }
+            }
+            int excess = removable.size() - copyLimit;
+            if (excess <= 0) {
+                return;
+            }
+            logger.info("Template [{}] has [{}] removable secondary storage copies in zone [{}], limit is [{}]; removing [{}] excess copies.",
+                    template.getUniqueName(), removable.size(), zoneId, copyLimit, excess);
+            for (int i = 0; i < excess; i++) {
+                DataStore ds = _storeMgr.getDataStore(removable.get(i).getDataStoreId(), DataStoreRole.Image);
+                try {
+                    deleteTemplateAsync(_templateFactory.getTemplate(templateId, ds));
+                    logger.info("Removed excess copy of template [{}] from image store [{}] to honor the secondary storage copy limit.",
+                            template.getUniqueName(), ds.getName());
+                } catch (Exception e) {
+                    logger.warn("Failed to remove excess copy of template [{}] from image store [{}]: {}",
+                            template.getUniqueName(), ds, e.getMessage());
+                }
+            }
+        } finally {
+            lock.unlock();
+            lock.releaseRef();
+        }
+    }
+
     protected boolean shouldDownloadTemplateToStore(VMTemplateVO template, DataStore store) {
         Long zoneId = store.getScope().getScopeId();
         DataStore directedStore = _tmpltMgr.verifyHeuristicRulesForZone(template, zoneId);
@@ -304,6 +391,12 @@ protected boolean shouldDownloadTemplateToStore(VMTemplateVO template, DataStore
             return false;
         }
 
+        if (zoneId != null && hasReachedSecStorageCopyLimit(template, zoneId)) {
+            logger.info("Skipping sync of template [{}] to image store [{}]: zone [{}] has reached the configured copy limit.",
+                    template.getUniqueName(), store.getName(), zoneId);
+            return false;
+        }
+
         if (template.isPublicTemplate()) {
             logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is public.", template.getUniqueName(),
                     store.getName());
@@ -531,10 +624,13 @@ public void handleTemplateSync(DataStore store) {
                                         && tmpltStore.getState() == State.Ready
                                         && tmpltStore.getInstallPath() == null) {
                                     logger.info("Keep fake entry in template store table for migration of previous NFS to object store");
-                                } else {
+                                } else if (tmpltStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED
+                                        || tmpltStore.getState() == State.Ready) {
                                     logger.info("Removing leftover template {} entry from template store table", tmplt);
-                                    // remove those leftover entries
                                     _vmTemplateStoreDao.remove(tmpltStore.getId());
+                                } else {
+                                    logger.debug("Template {} entry on store {} is in pre-download state ({}/{}); not treating as leftover.",
+                                            tmplt, store, tmpltStore.getState(), tmpltStore.getDownloadState());
                                 }
                             }
                         }
@@ -580,6 +676,12 @@ public void handleTemplateSync(DataStore store) {
                         }
                     }
 
+                    if (zoneId != null) {
+                        for (VMTemplateVO tmplt : allTemplates) {
+                            enforceSecStorageCopyLimit(tmplt.getId(), zoneId);
+                        }
+                    }
+
                     for (String uniqueName : templateInfos.keySet()) {
                         TemplateProp tInfo = templateInfos.get(uniqueName);
                         if (_tmpltMgr.templateIsDeleteable(tInfo.getId())) {

server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java

@@ -19,10 +19,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
@@ -292,9 +292,10 @@ protected void createTemplateWithinZones(TemplateProfile profile, VMTemplateVO t
 
             if (imageStore == null) {
                 List<DataStore> imageStores = getImageStoresThrowsExceptionIfNotFound(zoneId, profile);
-                standardImageStoreAllocation(imageStores, template);
+                standardImageStoreAllocation(imageStores, template, zoneId);
             } else {
-                validateSecondaryStorageAndCreateTemplate(List.of(imageStore), template, null);
+                int copyLimit = getSecStorageCopyLimit(template, zoneId);
+                validateSecondaryStorageAndCreateTemplate(List.of(imageStore), template, new HashMap<>(), copyLimit);
             }
         }
     }
@@ -307,17 +308,17 @@ protected List<DataStore> getImageStoresThrowsExceptionIfNotFound(long zoneId, T
         return imageStores;
     }
 
-    protected void standardImageStoreAllocation(List<DataStore> imageStores, VMTemplateVO template) {
-        Set<Long> zoneSet = new HashSet<Long>();
+    protected void standardImageStoreAllocation(List<DataStore> imageStores, VMTemplateVO template, long zoneId) {
+        int copyLimit = getSecStorageCopyLimit(template, zoneId);
         Collections.shuffle(imageStores);
-        validateSecondaryStorageAndCreateTemplate(imageStores, template, zoneSet);
+        validateSecondaryStorageAndCreateTemplate(imageStores, template, new HashMap<>(), copyLimit);
     }
 
-    protected void validateSecondaryStorageAndCreateTemplate(List<DataStore> imageStores, VMTemplateVO template, Set<Long> zoneSet) {
+    protected void validateSecondaryStorageAndCreateTemplate(List<DataStore> imageStores, VMTemplateVO template, Map<Long, Integer> zoneCopyCount, int copyLimit) {
         for (DataStore imageStore : imageStores) {
             Long zoneId = imageStore.getScope().getScopeId();
 
-            if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneSet, isPrivateTemplate(template))) {
+            if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneCopyCount, copyLimit)) {
                 continue;
             }
 

server/src/main/java/com/cloud/template/TemplateAdapterBase.java

@@ -20,11 +20,9 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 
 import javax.inject.Inject;
 
@@ -169,7 +167,11 @@ protected DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zone
         return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template);
     }
 
-    protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId, Set<Long> zoneSet, boolean isTemplatePrivate) {
+    protected int getSecStorageCopyLimit(VMTemplateVO template, long zoneId) {
+        return templateMgr.getSecStorageCopyLimit(template, zoneId);
+    }
+
+    protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId, Map<Long, Integer> zoneCopyCount, int copyLimit) {
         if (zoneId == null) {
             logger.warn(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", imageStore));
             return false;
@@ -191,19 +193,13 @@ protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId
             return false;
         }
 
-        if (zoneSet == null) {
-            logger.info(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage of zone [%s].", zone));
-            return true;
-        }
-
-        if (isTemplatePrivate && zoneSet.contains(zoneId)) {
-            logger.info(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; therefore, image store [%s] will be skipped.",
-                    zone, imageStore));
+        int currentCount = zoneCopyCount.getOrDefault(zoneId, 0);
+        if (copyLimit > 0 && currentCount >= copyLimit) {
+            logger.info("Copy limit of {} reached for zone [{}]; skipping image store [{}].", copyLimit, zone, imageStore);
             return false;
         }
 
-        logger.info(String.format("Private template will be allocated in image store [%s] in zone [%s].", imageStore, zone));
-        zoneSet.add(zoneId);
+        zoneCopyCount.put(zoneId, currentCount + 1);
         return true;
     }
 
@@ -212,12 +208,13 @@ protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId
      * {@link TemplateProfile#getZoneIdList()}.
      */
     protected void postUploadAllocation(List<DataStore> imageStores, VMTemplateVO template, List<TemplateOrVolumePostUploadCommand> payloads) {
-        Set<Long> zoneSet = new HashSet<>();
+        Map<Long, Integer> zoneCopyCount = new HashMap<>();
         Collections.shuffle(imageStores);
         for (DataStore imageStore : imageStores) {
             Long zoneId_is = imageStore.getScope().getScopeId();
+            int copyLimit = zoneId_is == null ? 0 : getSecStorageCopyLimit(template, zoneId_is);
 
-            if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneSet, isPrivateTemplate(template))) {
+            if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneCopyCount, copyLimit)) {
                 continue;
             }
 
@@ -254,15 +251,6 @@ protected void postUploadAllocation(List<DataStore> imageStores, VMTemplateVO te
         }
     }
 
-    protected boolean isPrivateTemplate(VMTemplateVO template){
-        // if public OR featured OR system template
-        if (template.isPublicTemplate() || template.isFeatured() || template.getTemplateType() == TemplateType.SYSTEM) {
-            return false;
-        } else {
-            return true;
-        }
-    }
-
     @Override
     public boolean stop() {
         return true;

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -1693,6 +1693,12 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
             _launchPermissionDao.removeAllPermissions(id);
             _messageBus.publish(_name, TemplateManager.MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT, PublishScope.LOCAL, template.getId());
         }
+
+        if (isPublic != null || isFeatured != null || "reset".equalsIgnoreCase(operation)) {
+            for (VMTemplateZoneVO templateZone : _tmpltZoneDao.listByTemplateId(template.getId())) {
+                _tmpltSvr.enforceSecStorageCopyLimit(template.getId(), templateZone.getZoneId());
+            }
+        }
         return true;
     }
 
@@ -2177,6 +2183,20 @@ public List<DataStore> getImageStoreByTemplate(long templateId, Long zoneId) {
         return stores;
     }
 
+    @Override
+    public int getSecStorageCopyLimit(VMTemplateVO template, long zoneId) {
+        if (template == null) {
+            return 0;
+        }
+        TemplateType type = template.getTemplateType();
+        if (type == TemplateType.SYSTEM || type == TemplateType.ROUTING || type == TemplateType.BUILTIN) {
+            return 0;
+        }
+        return template.isPublicTemplate()
+                ? PublicTemplateSecStorageCopy.valueIn(zoneId)
+                : PrivateTemplateSecStorageCopy.valueIn(zoneId);
+    }
+
     @Override
     @ActionEvent(eventType = EventTypes.EVENT_ISO_UPDATE, eventDescription = "Updating ISO", async = false)
     public VMTemplateVO updateTemplate(UpdateIsoCmd cmd) {
@@ -2493,7 +2513,9 @@ public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] {AllowPublicUserTemplates,
                 TemplatePreloaderPoolSize,
                 ValidateUrlIsResolvableBeforeRegisteringTemplate,
-                TemplateDeleteFromPrimaryStorage};
+                TemplateDeleteFromPrimaryStorage,
+                PublicTemplateSecStorageCopy,
+                PrivateTemplateSecStorageCopy};
     }
 
     public List<TemplateAdapter> getTemplateAdapters() {