[ADH-7056] Support option for switching between Ozone and HDFS modes

build-project.sh

@@ -3,5 +3,5 @@ set -e
 
 HDFS_VERSION=$1
 
-mvn install:install-file -pl smart-tests -Dos.arch=x86_64
-mvn clean install -Pdist,web-ui,hadoop-"${HDFS_VERSION}" -DskipTests -Dos.arch=x86_64
+mvn install:install-file -pl smart-tests
+mvn clean install -Pdist,web-ui,hadoop-"${HDFS_VERSION}" -DskipTests

docs/ssm-deployment-guide.md

@@ -532,7 +532,7 @@ Note: To make the scripts work, you have to set up SSH password-less connections
 ```
    2017-07-15 00:38:28,619 INFO org.smartdata.hdfs.HdfsStatesUpdateService.init 68: Initializing ...
    2017-07-15 00:38:29,350 ERROR org.smartdata.hdfs.HdfsStatesUpdateService.checkAndMarkRunning 138: Unable to lock 'mover', please stop 'mover' first.
-   2017-07-15 00:38:29,350 INFO org.smartdata.server.engine.StatesManager.initStatesUpdaterService 180: Failed to create states updater service.
+   2017-07-15 00:38:29,350 INFO org.smartdata.server.engine.HdfsStatesManager.initStatesUpdaterService 180: Failed to create states updater service.
 ```
 
 Notes

smart-action/src/main/java/org/smartdata/action/AbstractActionFactory.java

@@ -17,44 +17,66 @@
  */
 package org.smartdata.action;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
 import org.smartdata.action.annotation.ActionSignature;
 
-import java.util.Collections;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
 
 /**
  * A common action factory for action providers to use.
  */
+@Slf4j
 public abstract class AbstractActionFactory implements ActionFactory {
-  static final Logger LOG = LoggerFactory.getLogger(AbstractActionFactory.class);
-  private static Map<String, Class<? extends SmartAction>> supportedActions = new HashMap<>();
-
-  static {
-    addAction(EchoAction.class);
-    addAction(SleepAction.class);
-    addAction(SyncAction.class);
-    addAction(ExecAction.class);
-  }
+  private static final List<Class<? extends SmartAction>> COMMON_ACTIONS = Arrays.asList(
+      EchoAction.class,
+      SleepAction.class,
+      ExecAction.class
+  );
+
+  @Override
+  public Map<String, Class<? extends SmartAction>> getSupportedActions() {
+    Map<String, Class<? extends SmartAction>> supportedActions = new HashMap<>();
+    COMMON_ACTIONS.forEach(
+        actionClass -> addActionInfo(supportedActions, actionClass));
+    supportedActionClasses().forEach(
+        actionClass -> addActionInfo(supportedActions, actionClass));
 
-  protected static void addAction(Class<? extends SmartAction> actionClass) {
-    ActionSignature actionSignature = actionClass.getAnnotation(ActionSignature.class);
-    if (actionSignature != null) {
-      String actionId = actionSignature.actionId();
-      if (!supportedActions.containsKey(actionId)) {
-        supportedActions.put(actionId, actionClass);
-      } else {
-        LOG.error("There is already an Action registered with id {}.", actionId);
-      }
-    } else {
-      LOG.error("Action {} does not has an ActionSignature.", actionClass.getName());
-    }
+    return supportedActions;
   }
 
   @Override
-  public Map<String, Class<? extends SmartAction>> getSupportedActions() {
-    return Collections.unmodifiableMap(supportedActions);
+  public Set<ActionMetadata> getActionMetadata() {
+    Set<ActionMetadata> actionMetadata = new HashSet<>();
+    COMMON_ACTIONS.forEach(action ->
+        toActionMetadata(action).ifPresent(actionMetadata::add));
+    supportedActionClasses().forEach(action ->
+        toActionMetadata(action).ifPresent(actionMetadata::add));
+
+    return actionMetadata;
+  }
+
+  protected abstract List<Class<? extends SmartAction>> supportedActionClasses();
+
+  private Optional<ActionMetadata> toActionMetadata(Class<? extends SmartAction> actionClass) {
+    return actionSignature(actionClass)
+        .map(signature -> new ActionMetadata(signature.actionId(), signature.usage()));
+  }
+
+  private void addActionInfo(
+      Map<String, Class<? extends SmartAction>> supportedActions,
+      Class<? extends SmartAction> actionClass) {
+    actionSignature(actionClass)
+        .map(ActionSignature::actionId)
+        .ifPresent(actionId -> supportedActions.put(actionId, actionClass));
+  }
+
+  private Optional<ActionSignature> actionSignature(Class<? extends SmartAction> actionClass) {
+    return Optional.ofNullable(actionClass.getAnnotation(ActionSignature.class));
   }
 }

smart-action/src/main/java/org/smartdata/action/ActionFactory.java

@@ -18,6 +18,7 @@
 package org.smartdata.action;
 
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Action factory interface. Either built-in or user defined actions will be
@@ -30,4 +31,6 @@ public interface ActionFactory {
    * @return supported actions
    */
   Map<String, Class<? extends SmartAction>> getSupportedActions();
+
+  Set<ActionMetadata> getActionMetadata();
 }

smart-action/src/main/java/org/smartdata/action/ActionMetadata.java

@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.smartdata.action;
+
+import lombok.Data;
+
+@Data
+public class ActionMetadata {
+  private final String name;
+  private final String usage;
+}

smart-action/src/main/java/org/smartdata/action/ActionRegistry.java

@@ -17,73 +17,57 @@
  */
 package org.smartdata.action;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.smartdata.action.annotation.ActionSignature;
-import org.smartdata.model.ActionDescriptor;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
-import java.util.ServiceConfigurationError;
-import java.util.ServiceLoader;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * Actions registry. Singleton.
  */
+@Slf4j
 public class ActionRegistry {
-  static final Logger LOG = LoggerFactory.getLogger(ActionRegistry.class);
-  private static final Map<String, Class<? extends SmartAction>> ACTIONS = new ConcurrentHashMap<>();
+  @Getter
+  private final Set<ActionMetadata> actionMetadata;
+  private final Map<String, Class<? extends SmartAction>> actions;
 
-  static {
-    try {
-      ServiceLoader<ActionFactory> actionFactories = ServiceLoader.load(ActionFactory.class);
-      for (ActionFactory fact : actionFactories) {
-        ACTIONS.putAll(fact.getSupportedActions());
-      }
-    } catch (ServiceConfigurationError e) {
-      LOG.error("Load actions failed from factory");
-    }
-  }
+  public ActionRegistry(Collection<ActionFactory> factories) {
+    this.actions = new HashMap<>();
+    this.actionMetadata = new HashSet<>();
+
+    factories.stream()
+        .map(ActionFactory::getSupportedActions)
+        .forEach(actions::putAll);
 
-  public static Set<String> registeredActions() {
-    return Collections.unmodifiableSet(ACTIONS.keySet());
+    factories.stream()
+        .map(ActionFactory::getActionMetadata)
+        .forEach(actionMetadata::addAll);
   }
 
-  public static boolean registeredAction(String name) {
-    return ACTIONS.containsKey(name);
+  public Set<String> registeredActions() {
+    return actions.keySet();
   }
 
-  public static List<ActionDescriptor> supportedActions() {
-    List<ActionDescriptor> actionDescriptors = new ArrayList<>();
-    for (Class<? extends SmartAction> clazz : ACTIONS.values()) {
-      ActionSignature signature = clazz.getAnnotation(ActionSignature.class);
-      if (signature != null) {
-        actionDescriptors.add(fromSignature(signature));
-      }
-    }
-    return actionDescriptors;
+  public boolean isRegistered(String name) {
+    return actions.containsKey(name);
   }
 
-  public static SmartAction createAction(String name) throws ActionException {
-    if (!registeredAction(name)) {
+  public SmartAction createAction(String name) throws ActionException {
+    if (!isRegistered(name)) {
       throw new ActionException("Unregistered action " + name);
     }
+
     try {
-      SmartAction smartAction = ACTIONS.get(name).newInstance();
+      SmartAction smartAction = actions.get(name).newInstance();
       smartAction.setName(name);
       return smartAction;
     } catch (Exception e) {
-      LOG.error("Create {} action failed", name, e);
+      log.error("Create {} action failed", name, e);
       throw new ActionException(e);
     }
   }
-
-  private static ActionDescriptor fromSignature(ActionSignature signature) {
-    return new ActionDescriptor(
-        signature.actionId(), signature.displayName(), signature.usage(), signature.description());
-  }
 }

smart-agent/src/main/java/org/smartdata/agent/SmartAgent.java

@@ -78,6 +78,7 @@
 import static org.smartdata.conf.SmartConfKeys.SMART_AGENT_MASTER_CONNECT_TIMEOUT_MS_KEY;
 import static org.smartdata.conf.SmartConfKeys.SMART_CMDLET_EXECUTORS_DEFAULT;
 import static org.smartdata.conf.SmartConfKeys.SMART_CMDLET_EXECUTORS_KEY;
+import static org.smartdata.server.utils.ConfigUtil.enrichSmartConf;
 
 public class SmartAgent implements StatusReporter {
   private static final String NAME = "SmartAgent";
@@ -102,7 +103,7 @@ public SmartAgent(SmartConf smartConfig) throws IOException {
     LOG.info("Agent address: {}", agentAddress);
     this.akkaConfig = AgentUtils.overrideRemoteAddress(
         ConfigFactory.load(AgentConstants.AKKA_CONF_FILE), agentAddress);
-    HadoopUtil.setSmartConfByHadoop(smartConfig);
+    enrichSmartConf(smartConfig);
 
     this.smartConfig = smartConfig;
     this.httpServer = new SmartAgentHttpServer(smartConfig,

smart-common/src/main/java/org/smartdata/AbstractService.java

@@ -32,8 +32,4 @@ public AbstractService() {
   public AbstractService(SmartContext context) {
     this.context = context;
   }
-
-  public boolean inSafeMode() {
-    return false;
-  }
 }

smart-common/src/main/java/org/smartdata/SmartConstants.java

@@ -44,10 +44,23 @@ public class SmartConstants {
   public static final String SMART_FILE_CHECKSUM_XATTR_NAME = "user.checksum";
 
   public static final String FS_HDFS_IMPL = "fs.hdfs.impl";
+  public static final String FS_OFS_IMPL = "fs.ofs.impl";
+  public static final String FS_O3FS_IMPL = "fs.o3fs.impl";
   public static final String SMART_FILE_SYSTEM =
       "org.smartdata.hadoop.filesystem.SmartFileSystem";
+  public static final String SMART_OFS =
+      "org.apache.hadoop.fs.ozone.SmartRootedOzoneFileSystem";
+  public static final String SMART_O3FS =
+      "org.apache.hadoop.fs.ozone.SmartOzoneFileSystem";
+
   public static final String DISTRIBUTED_FILE_SYSTEM =
       "org.apache.hadoop.hdfs.DistributedFileSystem";
+  public static final String OFS =
+      "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem";
+  public static final String O3FS =
+      "org.apache.hadoop.fs.ozone.OzoneFileSystem";
+
+
 
   public static final String REPLICATION_CODEC_NAME = "replication";
 }

smart-common/src/main/java/org/smartdata/SmartService.java

@@ -50,4 +50,8 @@ enum State {
    * @throws IOException
    */
   void stop() throws IOException;
+
+  default boolean inSafeMode() {
+    return false;
+  }
 }