Beta branch refactoring

pom.xml

@@ -8,6 +8,7 @@
   <name>cfe_39</name>
   <properties>
     <changelist>-SNAPSHOT</changelist>
+    <cnf_01.version>1.0.0</cnf_01.version>
     <hadoop.version>3.3.6</hadoop.version>
     <java.version>1.8</java.version>
     <maven.compiler.source>1.8</maven.compiler.source>
@@ -78,6 +79,12 @@
       <artifactId>rlo_06</artifactId>
       <version>9.0.1</version>
     </dependency>
+    <!-- Configuration -->
+    <dependency>
+      <groupId>com.teragrep</groupId>
+      <artifactId>cnf_01</artifactId>
+      <version>${cnf_01.version}</version>
+    </dependency>
     <!-- kafka -->
     <dependency>
       <groupId>org.apache.kafka</groupId>
@@ -199,9 +206,14 @@
             <exclude>src/test/resources/broken.application.properties</exclude>
             <exclude>src/test/resources/valid.application.properties</exclude>
             <exclude>src/test/resources/failProcessing.application.properties</exclude>
+            <exclude>src/test/resources/largeFile.application.properties</exclude>
             <exclude>rpm/resources/config.jaas</exclude>
             <exclude>rpm/resources/log4j2.properties</exclude>
             <exclude>rpm/resources/application.properties</exclude>
+            <exclude>rpm/resources/ingress.properties</exclude>
+            <exclude>rpm/resources/egress.properties</exclude>
+            <exclude>src/test/resources/valid.hdfs.properties</exclude>
+            <exclude>src/test/resources/valid.kafka.properties</exclude>
             <exclude>rpm/resources/cfe_39.service</exclude>
             <exclude>rpm/rpm.pom.xml</exclude>
             <exclude>src/main/java/com/teragrep/cfe_39/avro/SyslogRecord.java</exclude>
@@ -380,6 +392,218 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-checkstyle-plugin</artifactId>
+        <version>3.5.0</version>
+        <executions>
+          <!-- These are all errors that will fail the build if triggered -->
+          <execution>
+            <id>scan-errors</id>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>process-classes</phase>
+            <configuration>
+              <violationSeverity>error</violationSeverity>
+              <logViolationsToConsole>true</logViolationsToConsole>
+              <failOnViolation>true</failOnViolation>
+              <failsOnError>false</failsOnError>
+              <checkstyleRules>
+                <module name="Checker">
+                  <module name="TreeWalker">
+                    <module name="ClassMemberImpliedModifier"></module>
+                    <module name="CovariantEquals"></module>
+                    <module name="DefaultComesLast"></module>
+                    <module name="EmptyBlock"></module>
+                    <module name="EmptyCatchBlock"></module>
+                    <module name="EmptyStatement"></module>
+                    <module name="EqualsAvoidNull"></module>
+                    <module name="EqualsHashCode"></module>
+                    <module name="FallThrough"></module>
+                    <module name="FinalClass"></module>
+                    <module name="HiddenField">
+                      <property name="ignoreConstructorParameter" value="true"></property>
+                      <property name="ignoreSetter" value="true"></property>
+                    </module>
+                    <module name="IllegalCatch"></module>
+                    <module name="IllegalImport"></module>
+                    <module name="IllegalThrows"></module>
+                    <module name="IllegalToken"></module>
+                    <module name="IllegalType"></module>
+                    <module name="InnerAssignment"></module>
+                    <module name="InterfaceMemberImpliedModifier"></module>
+                    <module name="MissingOverride"></module>
+                    <module name="MissingSwitchDefault"></module>
+                    <module name="ModifiedControlVariable">
+                      <property name="skipEnhancedForLoopVariable" value="true"></property>
+                    </module>
+                    <module name="ModifierOrder"></module>
+                    <module name="MutableException"></module>
+                    <module name="NeedBraces"></module>
+                    <module name="NestedForDepth">
+                      <property name="max" value="2"></property>
+                    </module>
+                    <module name="NestedTryDepth"></module>
+                    <module name="NoClone"></module>
+                    <module name="NoFinalizer"></module>
+                    <module name="OneTopLevelClass"></module>
+                    <module name="PackageDeclaration"></module>
+                    <module name="PackageName">
+                      <property name="format" value="^com\.teragrep\.[a-z]{3}_\d{2}(?:.[a-zA-Z]\w*)*$"></property>
+                    </module>
+                    <module name="ReturnCount">
+                      <property name="max" value="5"></property>
+                    </module>
+                    <module name="StringLiteralEquality"></module>
+                    <module name="SuperClone"></module>
+                    <module name="SuperFinalize"></module>
+                    <module name="TypeName"></module>
+                    <module name="UpperEll"></module>
+                    <module name="VisibilityModifier">
+                      <property name="allowPublicFinalFields" value="true"></property>
+                    </module>
+                  </module>
+                  <module name="Translation"></module>
+                  <module name="UniqueProperties"></module>
+                  <module name="BeforeExecutionExclusionFileFilter">
+                    <property name="fileNamePattern" value="SyslogRecord\.java$"></property>
+                  </module>
+                </module>
+              </checkstyleRules>
+            </configuration>
+          </execution>
+          <!-- These are warnings but will not fail the build -->
+          <execution>
+            <id>scan-warnings</id>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>process-classes</phase>
+            <configuration>
+              <violationSeverity>warning</violationSeverity>
+              <logViolationsToConsole>true</logViolationsToConsole>
+              <failOnViolation>false</failOnViolation>
+              <failsOnError>false</failsOnError>
+              <checkstyleRules>
+                <module name="Checker">
+                  <module name="TreeWalker">
+                    <module name="ArrayTypeStyle">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="AvoidNestedBlocks">
+                      <property name="allowInSwitchCase" value="true"></property>
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="AvoidNoArgumentSuperConstructorCall">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="AvoidStarImport">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="AvoidStaticImport">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="DeclarationOrder">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="FinalLocalVariable">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="FinalParameters">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="MagicNumber">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="MissingDeprecated">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="MultipleVariableDeclarations">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="NestedForDepth">
+                      <property name="max" value="1"></property>
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="NestedIfDepth">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="NoArrayTrailingComma">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="NoCodeInFile">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="NoEnumTrailingComma">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="OneStatementPerLine">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="OuterTypeFilename">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="ParameterAssignment">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="RedundantImport">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="RequireThis">
+                      <property name="checkFields" value="false"></property>
+                      <property name="checkMethods" value="false"></property>
+                      <property name="validateOnlyOverlapping" value="true"></property>
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="ReturnCount">
+                      <property name="max" value="1"></property>
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="SimplifyBooleanExpression">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="SimplifyBooleanReturn">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="UnnecessarySemicolonAfterOuterTypeDeclaration">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="UnnecessarySemicolonAfterTypeMemberDeclaration">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="UnnecessarySemicolonInEnumeration">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="UnnecessarySemicolonInTryWithResources">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="UnusedLocalVariable">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="VariableDeclarationUsageDistance">
+                      <property name="severity" value="warning"></property>
+                    </module>
+                    <module name="VisibilityModifier">
+                      <property name="allowPublicFinalFields" value="false"></property>
+                      <property name="severity" value="warning"></property>
+                    </module>
+                  </module>
+                  <module name="NewlineAtEndOfFile">
+                    <property name="severity" value="warning"></property>
+                  </module>
+                  <module name="OrderedProperties">
+                    <property name="severity" value="warning"></property>
+                  </module>
+                  <module name="BeforeExecutionExclusionFileFilter">
+                    <property name="fileNamePattern" value="SyslogRecord\.java$"></property>
+                  </module>
+                </module>
+              </checkstyleRules>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 </project>

rpm/resources/application.properties

@@ -1,52 +1,16 @@
-# Kafka security configuration file
-java.security.auth.login.config=/opt/teragrep/cfe_39/etc/config.jaas
 # Logger settings
 log4j2.configurationFile=/opt/teragrep/cfe_39/etc/log4j2.properties
+# hdfs settings
+egress.configurationFile=/opt/teragrep/cfe_39/etc/egress.properties
+# kafka settings
+ingress.configurationFile=/opt/teragrep/cfe_39/etc/ingress.properties
 # What topics are searched from kafka, regex
 queueTopicPattern=^testConsumerTopic-*$
-# Number of consumers created to the consumer groups
-numOfConsumers=2
-# Kafka bootstrap servers
-consumer.bootstrap.servers=test
-# Offset, should not be touched
-consumer.auto.offset.reset=earliest
-# Autocommit, should not be touched
-consumer.enable.auto.commit=false
-# Consumer group id, this is to track the progress of reading hte topic
-consumer.group.id=cfe_39
-# Used security protocol and mechanism
-consumer.security.protocol=SASL_PLAINTEXT
-consumer.sasl.mechanism=PLAIN
-# Maximum records per batch, note that too big number will cause massive load and can cause timeouts to trigger
-consumer.max.poll.records=500
-# How much data can be fetched in one go
-consumer.fetch.max.bytes=1073741820
-# How long for request before timing out. Note that too big max poll records size can cause this to trigger
-consumer.request.timeout.ms=300000
-consumer.max.poll.interval.ms=300000
-# For testing only, remove for prod.
-consumer.useMockKafkaConsumer=true
 # Directory where AVRO files are constructed for HDFS
 queueDirectory=/opt/teragrep/cfe_39/etc/AVRO/
-# The maximum file size for AVRO-files that are to be stored in HDFS database.
-maximumFileSize=60800000
 # Boolean for deciding if records not in RFC5424 should be skipped or not.
 skipNonRFC5424Records=true
 # Boolean for deciding if empty RFC5424 records should be skipped or not.
 skipEmptyRFC5424Records=true
-# HDFS pruning offset, prunes files older than the given milliseconds.
-pruneOffset=172800000
-# HDFS uri
-hdfsuri=hdfs://localhost:45937/
-# Kerberos
-java.security.krb5.kdc=test
-java.security.krb5.realm=test
-hadoop.security.authentication=test
-hadoop.security.authorization=test
-dfs.namenode.kerberos.principal.pattern=test
-KerberosKeytabUser=test
-KerberosKeytabPath=test
-dfs.client.use.datanode.hostname=false
-kerberosLoginAutorenewal=true
-dfs.data.transfer.protection=test
-dfs.encrypt.data.transfer.cipher.suites=test
+# timeout modifier for when the consumer's cache of intermediate results are flushed to HDFS
+consumerTimeout=600000

rpm/resources/egress.properties

@@ -0,0 +1,20 @@
+# HDFS pruning, use 157784760000 value while testing HDFS writes to ensure the test records are not pruned. 157784760000L
+pruneOffset=157784760000
+# HDFS uri
+hdfsuri=hdfs://localhost:45937/
+# HDFS path
+hdfsPath=hdfs:///opt/teragrep/cfe_39/srv/
+# Kerberos
+java.security.krb5.kdc=test
+java.security.krb5.realm=test
+hadoop.security.authentication=kerberos
+hadoop.security.authorization=test
+dfs.namenode.kerberos.principal.pattern=test
+KerberosKeytabUser=test
+KerberosKeytabPath=test
+dfs.client.use.datanode.hostname=false
+hadoop.kerberos.keytab.login.autorenewal.enabled=true
+dfs.data.transfer.protection=test
+dfs.encrypt.data.transfer.cipher.suites=test
+# The maximum file size for AVRO-files that are to be stored in HDFS database.
+maximumFileSize=3000

rpm/resources/ingress.properties

@@ -0,0 +1,24 @@
+# Kafka security configuration file
+java.security.auth.login.config=/opt/teragrep/cfe_39/etc/config.jaas
+# Kafka bootstrap servers
+bootstrap.servers=test
+# Offset, should not be touched
+auto.offset.reset=earliest
+# Autocommit, should not be touched
+enable.auto.commit=false
+# Consumer group id, this is to track the progress of reading hte topic
+group.id=cfe_39
+# Used security protocol and mechanism
+security.protocol=SASL_PLAINTEXT
+sasl.mechanism=PLAIN
+# Maximum records per batch, note that too big number will cause massive load and can cause timeouts to trigger
+max.poll.records=500
+# How much data can be fetched in one go
+fetch.max.bytes=1073741820
+# How long for request before timing out. Note that too big max poll records size can cause this to trigger
+request.timeout.ms=300000
+max.poll.interval.ms=300000
+# For testing only
+useMockKafkaConsumer=true
+# Number of consumers created to the consumer groups
+numOfConsumers=2