CODE EXPERIENCE BETWEEN 2018-2020

SLACK MESSAGE POSTING:

Create App, enable messaging option ,create incoming web hook url. add app to the channel you want to send message .

https://ghx.slack.com/apps/A0F7XDUAZ-incoming-webhooks?tab=settings&next_id=0

AttachmentFields.java

package com.ghx.testframework.api.entity;


import lombok.AllArgsConstructor;

import lombok.Builder;

import lombok.Getter;

import lombok.Setter;


@AllArgsConstructor

@Builder(builderClassName = "Builder")

@Getter

@Setter

public class AttachmentFields {

private String text;

private String fallback;

private String color;

}

SlackMessage.java


package com.ghx.testframework.api.entity;


import java.util.List;


import lombok.AllArgsConstructor;

import lombok.Builder;

import lombok.Getter;

import lombok.Setter;


@AllArgsConstructor

@Builder(builderClassName = "Builder")

@Getter

@Setter

public class SlackMessage {


private String channel;

private String username;

private String text;

private String icon_emoji;

private String color;

private List<AttachmentFields> attachments;

}


SlackUtils.java

package com.ghx.testframework.api.util;


import java.io.IOException;


import org.apache.http.client.methods.HttpPost;

import org.apache.http.entity.StringEntity;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClients;


import com.fasterxml.jackson.databind.ObjectMapper;

import com.ghx.common.log.GHXLogger;

import com.ghx.common.log.GHXLoggerFactory;

import com.ghx.testframework.api.entity.SlackMessage;

import com.ghx.testframework.utils.enums.TFConstants;


public class SlackUtils {

private static final GHXLogger LOGGER = GHXLoggerFactory.getLogger(SlackUtils.class);


public static void sendMessage(SlackMessage message) {

String slackWebhookUrl = TFConstants.TEST_FRAMEWORK_SLACK_NOTIFICATION_URL.getValue();


CloseableHttpClient client = HttpClients.createDefault();

HttpPost httpPost = new HttpPost(slackWebhookUrl); (create incoming slack webhook URL and use the url for posting message)


try {

ObjectMapper objectMapper = new ObjectMapper();

String json = objectMapper.writeValueAsString(message);


StringEntity entity = new StringEntity(json);

httpPost.setEntity(entity);

httpPost.setHeader("Accept", TFConstants.APPLICATION_JSON.getValue());

httpPost.setHeader("Content-type", TFConstants.APPLICATION_JSON.getValue());


client.execute(httpPost);

client.close();

} catch (IOException e) {

LOGGER.debug("Exception while posting Message to Slack: {}", e);

}

}

}

Usage java class

    TFW_TESTRUN_NOTIFICATION_INCOMING_WEBHOOKURL("test-framework.testrun.notification.incoming.webhookurl"),

LOADTEST("Load Test - "),
CHANNEL_NAME("test-framework-testrun-notification"),
LOAD_TEST_NOTIFICATION("LOAD TEST NOTIFICATION"),
ICON_EMOJI(":traffic_light:"),
APPLICATION_JSON("application/json"),
TEST_FRAMEWORK_SLACK_NOTIFICATION_URL(testRunNotificationSlackWebHookUrl()),

THOUSAND("1000");

private static String testRunNotificationSlackWebHookUrl() {
return ConfigSvcFactory.singleton().getProperty(TFConfigKeys.TFW_TESTRUN_NOTIFICATION_INCOMING_WEBHOOKURL.getValue(), StringUtils.EMPTY);
}

List<AttachmentFields> attachmentFields = Lists.newArrayList();

AttachmentFields attachStartedFields = AttachmentFields.builder().text(TFConstants.LOADTEST.getValue()+" *"+result.getTestRunName()+"* started by *"+result.getCreated().getBy()+"* in *"+EC2Utils.determineEnvironmentType().trim()+". :rocket: *").color("#FFA500").build();

attachmentFields.add(attachStartedFields);


SlackMessage slackMessage = SlackMessage.builder().channel(TFConstants.CHANNEL_NAME.getValue())

.username(TFConstants.LOAD_TEST_NOTIFICATION.getValue())

.attachments(attachmentFields)

.icon_emoji(TFConstants.ICON_EMOJI.getValue()).build();

SlackUtils.sendMessage(slackMessage);


Simple Usage java class:


SlackMessage slackMessage = SlackMessage.builder()

            .channel("test-framework-testrun-notification")

            .username("Selvapriya Kannan")

            .text("Load Test for Scenario"+ testScenarioName +"Starting")

            .icon_emoji(":smiley_cat:")

            .build();

    SlackUtils.sendMessage(slackMessage);

***********************************************************************************

Bamboo build and Deployment plan:

1.Create a project with name "Test**Lambda"

2.Create a Plan "Test****plan" and provide the project.

In Repositories tab, add repositories ,give repo Name , server Name "stash", repository "bucket Name", branch "branch name", Advance Options check (Use shallow clones,check Enable repository cache on remote agent, check Use submodule, command Time out (minutes)180,

in change detection option, in include/exclude files "Include Only changes that matches","^.*test\/test-lambdas\/.*" (only look for changes in folder in test with folder test-lambdas.

web repository "Bit bucket server".

ADD JOBS"BUILD":

3.Configure or edit build plan, add Task "source Code Checkout", give task Description ,add Project in Repository(repository added above). check force clean build and save.

4.Maven Version Variable Extractor, give Task Description Name, Give variable definition "EGX_Version", check remove -snapshot from version if present.In scope select Result and save.

5.Add task "Artifactory Maven3 Configuration",give task description"name", Goals "clean install -DskipTests

-Dgrunt.publish=true" , build JDK-Coretto-1.8, executable "Maven 3.5", Environmental Variables"JAVA_OPTS="$JAVA_OPTS  -XX:+AggressiveHeap" ",check resolve artifacts from artifactory, Resolution select "https://ghxrepo.jfrog.io/ghxrepo", resolution repository "ghx-all", artifacts from artifactory, Resolution select "https://ghxrepo.jfrog.io/ghxrepo", Target repository "enhancement-snapshot", check Deploy maven Artifacts, Deployment Include patterns "TestFramework-Lambda-Deployment/*", check Capture Build info, click save.

6.Amazon s3 Object Configuration Task Description Name, Region "US West (Oregon)", Artifact "Local Files", source Local Path "test-framework/test-framework-lambdas/*", Target Bucket Name "testframework-lambda-deployment" ,Target Object Key Prefix (Virtual Directory)  "Corex-Testframework-Lambda/${bamboo.EGX_Version}/${bamboo.planRepository.branchName}/${bamboo.buildNumber}/ " Tags "[

"Key":"service",

"Value":"corex-test-framework"

},

{"Key":"owner",

"Value":"enhancedeliveryteam@ghx.com"

}

]"  Aws Security Credentials Source "Identity federation for AWS", connectors"bamboo-devtest", save.

7.ADD UPLOAD FILES JOBS -Upload Files: 

add task ---script: 

Interpreter "/bin/sh or cmd.exe", script location "Inline",  script body " #!/bin/bash -x


copyScripts () {

    mongo_ds_dir=$1


    mkdir -p "mongoscripts/${mongo_ds_dir}"


    mongo_ds_release_dir="${mongo_ds_dir}/release"

    mongo_ds_release_version_dir="${mongo_ds_release_dir}/$2"

    echo "Processing mongo scripts for ${mongo_ds_dir} from location ${mongo_ds_release_version_dir}"


    if [ -d "${mongo_ds_release_version_dir}" ]

    then

        echo "SUCCESS: Directory ${mongo_ds_release_version_dir} exists."

    else

        echo "WARN: Directory ${mongo_ds_release_version_dir} does not exists. Trying to find latest version that might exist in directory to upload."

        minor_version=`ls ${mongo_ds_release_dir}| grep "^$3" | awk -F'_' '{ print $2 }' | sort -n | tail -1` 

        echo "ls ${mongo_ds_release_dir}| grep "^$3" | awk -F'_' '{ print $2 }' | sort -n | tail -1"

        echo "minor version is ${minor_version}"

        mongo_ds_release_version_dir="${mongo_ds_release_dir}/$3_${minor_version}_0"

        echo "mongo_ds_release_version_dir is ${mongo_ds_release_version_dir}"

    fi


    echo "COPYING: file from ${mongo_ds_release_version_dir}"

    cp -a ${mongo_ds_release_version_dir}/. ./mongoscripts/${mongo_ds_dir}/

    #cp ${mongo_ds_release_version_dir}/runall*.js ./mongoscripts/${mongo_ds_dir}/runall.js

}


copyCustomScripts () {

    mkdir -p "mongoscripts/custom_update"    

    #echo "COPYING: js files from test-framework/test-framework-api/release/*.js to mongoscripts"

    find test-framework/test-framework-api/release/util/custom_updates/ -maxdepth 1 -type f -name "*.js" | xargs -i cp {} ./mongoscripts/custom_update/

}


egxbranchname=${bamboo.planRepository.branchName}

egxversion=${bamboo.EGX_Version}    

echo $egxversion

egx_version=$(echo $egxversion | sed 's/\./\_/g')

egx_version="v$egx_version"

echo "current version is $egx_version"


#major_version=`ls | grep "^v"| awk -F'_' '{ print $1 }' | sort | tail -1`

#echo "deduced major version is ${major_version}"

major_version=`echo ${egx_version} | awk -F'_' '{ print $1 }'`

echo "Major version from ${egx_version} is ${major_version}"


copyScripts 'event-broker-ds' ${egx_version} ${major_version}


copyScripts 'audit-ds' ${egx_version} ${major_version}


copyScripts 'statistics-ds' ${egx_version} ${major_version}


copyScripts 'audit-relationship-ds' ${egx_version} ${major_version}


copyScripts 'partner-service-monitoring' ${egx_version} ${major_version}


copyScripts 'configsvc/configsvc-api' ${egx_version} ${major_version}


copyScripts 'archive-ds' ${egx_version} ${major_version}


copyScripts 'test-framework/test-framework-api' ${egx_version} ${major_version}


copyScripts 'notification-framework/notification-ds' ${egx_version} ${major_version}


copyScripts 'business-transaction-ds' ${egx_version} ${major_version}


copyScripts 'eventbus/eventbus-api' ${egx_version} ${major_version}


copyCustomScripts


echo "Finished" "

Script 2:

cp -r clouddeployment/test-framework/elasticsearch .

cp -r clouddeployment/test-framework/replay .

In Trigger tab:

Add Trigger , add BitBucket Server repository triggered, trigger description name, check repository, save Trigger

IN Branch Tab:

Create Plan Branch check Manually, Delete Plan Branch "check After plan was deleted from repository", 5 days, After branch inactive in repository, 5 days, Notifications -> choose "Notify commiters and people who have favourited the created branch", trigger -> Branch triggers -> Same as defined in parent plan", and save.

IN permission tab:
can provide read, write ,edit ,delete, view access.

In Notification tab:

Failed Build and First Successful -> committers (users who have committed to the build),All build completed (to main people who is owner)

In Variables tab:

awsRegion : us-east-1, baseami: ami-01b5ebd032022c6e1, baseami_release: ami-4c67f42c, baseamiold:ami-65038105,   

In Audit Log tab:

Logs the changes in build plan.

In Others tab:
 Expiry-> check Override global build expiry configuration, what should be expired -> check build artifacts, check build logs ,Expire after 1 days, Force stop build -> check Force stop hanging build and clean up sub-processes. Allure Reporting -> check build report only for failed builds,Executable -> allure-2.7.0 and save.

In plan Details tab:

plan name , plan description , check plan enabled save.


DEPLOYMENT PLAN PROJECT:

Create project " Test-Lambda-Deploy", configure lambda-deploy Deployment plan, Edit environment, IN edit plans -> Add Environment Name and description, 

How you want to deploy


What tasks need to happen to make this deployment a success

clean working directory task-> task description, save.

Aws CloudFormation Stack Configuration

Task Description -> "Name", create stack check update stack, if it already exists,check Dont fail for no-op update.
Region -> US-West (Oregon), Stack Name->TFW-Lambda-CICD, Template source check URL, Template URL "https://testframework-lambda-deployment.s3-us-west-2.amazonaws.com/Corex-Testframework-Lambda/${bamboo.EGX_Version}/${bamboo.planRepository.branchName}/${bamboo.buildNumber}/test-framework/test-framework-lambdas/src/main/resources/testframework-lambda-deployment.json",
Parameters -> 
[
{
  "ParameterKey": "S3ParentLambdaFolder",
        "ParameterValue": "Corex-Testframework-Lambda/${bamboo.EGX_Version}/${bamboo.planRepository.branchName}/${bamboo.buildNumber}/test-framework/test-framework-lambdas"
},
{
"ParameterKey": "CurrentVersion",
"ParameterValue": "${bamboo.EGX_Version}"
},
{
"ParameterKey": "BuildNumber",
"ParameterValue": "${bamboo.buildNumber}"
}
]

Advance Options -> Tag-> [

"Key":"service",

"Value":"corex-test-framework"

},

{"Key":"owner",

"Value":"enhancedeliveryteam@ghx.com"

}

]

On Failure Actions -> Rollback, Stack Policy source -> URL, save 

Other environment settings

Edit Triggers: After Successful build plan ->Trigger description and save.

RELEASE VERSIONING:
Next release ->version -> release-1 , automatically increment with each new release numbers  check  last number in version, save.
In project permission tab: add user and permissions.
 In audit log tab: logs of deployment.

Edit deployment: 
Deployment Project details :
Name :
description:
Link to build plan

build plan -> Corex-Test-FrameWork-Lambda › Test-Framework-Lambda-Plan
choose "Use the main plan branch",save deployment project.

Lambda Deployment json template:

Template for adding lambda 

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Description": "Build TFW Delete Stack Lambda functions for Deleting older stacks",
  "Parameters": {
    "S3ParentLambdaFolder": {
      "Description": "Provide the S3 Folder of lambda source zip",
      "Type": "String"
    },
    "BuildNumber": {
      "Description": "Bamboo build number",
      "Type": "String",
      "ConstraintDescription": "must be a valid bamboo build number"
    },
    "CurrentVersion": {
      "Description": "Bamboo Current Version",
      "Type": "String",
      "ConstraintDescription": "must be a valid bamboo Version"
    }
  },
  "Mappings": {
    "RegionMap": {
      "us-west-2": {
        "S3Bucket": "testframework-lambda-deployment",
        "S3Key": "S3ParentLambdaFolder",
        "role": "lambda_s3_vpc_execution"
      }
    }
  },
  "Resources": {
    "TFWDeletestack": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Code": {
          "S3Bucket": "testframework-lambda-deployment",
          "S3Key": {
            "Fn::Join": [
              "",
              [
                {
                  "Ref": "S3ParentLambdaFolder"
                },
                "/target/tfw-delete-stacks.zip"
              ]
            ]
          }
        },
        "Description": "TFW Delete old stack",
        "FunctionName": "TFW_Delete_Stacks",
        "Handler": "src/main/resources/tfw-delete-stacks.lambda_handler",
        "Environment": {
          "Variables": {
            "deletion_per_week_on": "Saturday",
            "filter_for_deletion": "CXTF,TestFW-MINION,TestFW-GRU-DEV-CXTF,TestFW-Replay",
            "filter_service": "corex-test-framework",
            "max_holding_days": "3",
            "status_to_be_deleted": "ROLLBACK_COMPLETE"
          }
        },
        "MemorySize": 128,
        "Role": {
          "Fn::Join": [
            "",
            [
              "arn:aws:iam::",
              {
                "Ref": "AWS::AccountId"
              },
              ":role/",
              "testframework_lambda"
            ]
          ]
        },
        "Runtime": "python2.7",
        "Timeout": "120"
      }
    },
    "LambdaVersionForTFWDeletestack": {
      "Type": "AWS::Lambda::Version",
      "Properties": {
        "FunctionName": {
          "Ref": "TFWDeletestack"
        },
        "Description": "v1"
      }
    },
    "ScheduledRuleForTFWDeletestack": {
      "Type": "AWS::Events::Rule",
      "Properties": {
        "Description": "ScheduledRule for triggering every hour your application",
        "ScheduleExpression": "rate(1 day)",
        "State": "ENABLED",
        "Targets": [
          {
            "Arn": {
              "Fn::GetAtt": [
                "TFWDeletestack",
                "Arn"
              ]
            },
            "Id": "TargetFunctionV1"
          }
        ]
      }
    },
    "TFWDeletestackPermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Fn::GetAtt": [
            "TFWDeletestack",
            "Arn"
          ]
        },
        "Principal": "events.amazonaws.com",
        "SourceArn": {
          "Fn::GetAtt": [
            "ScheduledRuleForTFWDeletestack",
            "Arn"
          ]
        }
      }
    },
    "TFWESClusterLambda": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Code": {
          "S3Bucket": "testframework-lambda-deployment",
          "S3Key": {
            "Fn::Join": [
              "",
              [
                {
                  "Ref": "S3ParentLambdaFolder"
                },
                "/target/tfw-es-cluster-lambda.zip"
              ]
            ]
          }
        },
        "Tags": [
          {
            "Key": "delete",
            "Value": "true"
          }
        ],
        "Description": "TFW Delete Elastic Search Data Greater than provided value",
        "FunctionName": "TFW_ES_Cluster_Lambda",
        "Handler": "src/main/resources/tfw-es-cluster-lambda.lambda_handler",
        "Environment": {
          "Variables": {
            "TEST_FRAMEWORK_ES_CLUSTER_ALLOWED_INDICES_COUNT": "150",
            "TEST_FRAMEWORK_ES_CLUSTER_MAINTAINERS": "spkannan@ghx.com,rkmurugasamy@ghx.com,spalanichamy@ghx.com",
            "TEST_FRAMEWORK_ES_CLUSTER_URL": "https://search-test-framework-es7-5r3n4oyw2kddzfmns4oo4okr7m.us-west-2.es.amazonaws.com"
          }
        },
        "MemorySize": 128,
        "Role": {
          "Fn::Join": [
            "",
            [
              "arn:aws:iam::",
              {
                "Ref": "AWS::AccountId"
              },
              ":role/",
              "lambda_s3_vpc_execution"
            ]
          ]
        },
        "Runtime": "python2.7",
        "Timeout": "300",
        "VpcConfig": {
          "SecurityGroupIds": [
            "sg-e72bdb88"
          ],
          "SubnetIds": [
            "subnet-72c7d206",
            "subnet-3b192d7d",
            "subnet-821aefe7"
          ]
        }
      }
    },
    "LambdaVersionForTFWESClusterLambda": {
      "Type": "AWS::Lambda::Version",
      "Properties": {
        "FunctionName": {
          "Ref": "TFWESClusterLambda"
        },
        "Description": "v1"
      }
    },
    "ScheduledRuleForTFWESClusterLambda": {
      "Type": "AWS::Events::Rule",
      "Properties": {
        "Description": "ScheduledRule for triggering every 18 second in your application",
        "ScheduleExpression": "rate(1 day)",
        "State": "ENABLED",
        "Targets": [
          {
            "Arn": {
              "Fn::GetAtt": [
                "TFWESClusterLambda",
                "Arn"
              ]
            },
            "Id": "TargetFunctionV1"
          }
        ]
      }
    },
    "TFWESClusterLambdaPermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Fn::GetAtt": [
            "TFWESClusterLambda",
            "Arn"
          ]
        },
        "Principal": "events.amazonaws.com",
        "SourceArn": {
          "Fn::GetAtt": [
            "ScheduledRuleForTFWESClusterLambda",
            "Arn"
          ]
        }
      }
    },
    "TFWBuildStatus": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Code": {
          "S3Bucket": "testframework-lambda-deployment",
          "S3Key": {
            "Fn::Join": [
              "",
              [
                {
                  "Ref": "S3ParentLambdaFolder"
                },
                "/target/test-framework-lambdas-",
                {
                  "Ref": "CurrentVersion"
                },
                "-SNAPSHOT.jar"
              ]
            ]
          }
        },
        "Description": "TFW Get All Corex Build Number and Status",
        "FunctionName": "TFW_Build_Status",
        "Handler": "com.testframework.buildstatus.BuildStatusHandler::handleRequest",
        "Environment": {
          "Variables": {
            "bucketName": "ghx-deployment",
            "region": "us-east-1"
          }
        },
        "MemorySize": 2048,
        "Role": {
          "Fn::Join": [
            "",
            [
              "arn:aws:iam::",
              {
                "Ref": "AWS::AccountId"
              },
              ":role/",
              "lambda_s3_vpc_execution"
            ]
          ]
        },
        "Runtime": "java8",
        "Timeout": "60"
      }
    },
    "LambdaVersionForTFWBuildStatus": {
      "Type": "AWS::Lambda::Version",
      "Properties": {
        "FunctionName": {
          "Ref": "TFWBuildStatus"
        },
        "Description": "v1"
      }
    },
    "ScheduledRuleForTFWBuildStatus": {
      "Type": "AWS::Events::Rule",
      "Properties": {
        "Description": "ScheduledRule for triggering every 18 second in your application",
        "ScheduleExpression": "cron(0/18 * * * ? *)",
        "State": "ENABLED",
        "Targets": [
          {
            "Arn": {
              "Fn::GetAtt": [
                "TFWBuildStatus",
                "Arn"
              ]
            },
            "Id": "TargetFunctionV1",
            "Input": "{\"buildWithStatus\":{ \"path\" : \"\"},\"isFolderExtRequired\":\"false\"}"
          }
        ]
      }
    },
    "TFWBuildStatusPermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Fn::GetAtt": [
            "TFWBuildStatus",
            "Arn"
          ]
        },
        "Principal": "events.amazonaws.com",
        "SourceArn": {
          "Fn::GetAtt": [
            "ScheduledRuleForTFWBuildStatus",
            "Arn"
          ]
        }
      }
    },
    "TFWS3Deletion": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Code": {
          "S3Bucket": "testframework-lambda-deployment",
          "S3Key": {
            "Fn::Join": [
              "",
              [
                {
                  "Ref": "S3ParentLambdaFolder"
                },
                "/target/tfw-s3-temp-folder-deletion.zip"
              ]
            ]
          }
        },
        "Description": "TFW Delete s3 temp files and folders in load-s3-testframework bucket",
        "FunctionName": "TFW_S3_Deletion",
        "Handler": "src/main/resources/tfw-s3-deletion/tfw-s3-temp-folder-deletion.handler",
        "Environment": {
          "Variables": {
            "DAY_DIFF": "1",
            "PREFIX": "temp",
            "S3_BUCKET": "load-s3-testframework"
          }
        },
        "MemorySize": 512,
        "Role": {
          "Fn::Join": [
            "",
            [
              "arn:aws:iam::",
              {
                "Ref": "AWS::AccountId"
              },
              ":role/",
              "lambda_s3_vpc_execution"
            ]
          ]
        },
        "Runtime": "nodejs12.x",
        "Timeout": "120"
      }
    },
    "LambdaVersionForTFWS3Deletion": {
      "Type": "AWS::Lambda::Version",
      "Properties": {
        "FunctionName": {
          "Ref": "TFWS3Deletion"
        },
        "Description": "v1"
      }
    },
    "ScheduledRuleForTFWS3Deletion": {
      "Type": "AWS::Events::Rule",
      "Properties": {
        "Description": "Delete temp files and folders in load-s3-testframework bucket at Every day 10.30pm (EST)",
        "ScheduleExpression": "cron(30 4 * * ? *)",
        "State": "ENABLED",
        "Targets": [
          {
            "Arn": {
              "Fn::GetAtt": [
                "TFWS3Deletion",
                "Arn"
              ]
            },
            "Id": "TargetFunctionV1"
          }
        ]
      }
    },
    "TFWS3DeletionPermission": {
      "Type": "AWS::Lambda::Permission",
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Fn::GetAtt": [
            "TFWS3Deletion",
            "Arn"
          ]
        },
        "Principal": "events.amazonaws.com",
        "SourceArn": {
          "Fn::GetAtt": [
            "ScheduledRuleForTFWS3Deletion",
            "Arn"
          ]
        }
      }
    }
  }
}

create-zipfiles-for-lambda.sh:

zip target/tfw-delete-stacks.zip src/main/resources/tfw-delete-stacks.py

zip target/tfw-es-cluster-lambda.zip src/main/resources/tfw-es-cluster-lambda.py

zip -r target/tfw-s3-temp-folder-deletion.zip src/main/resources/tfw-s3-deletion


pom.xml:
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.ghx</groupId>
<artifactId>test-framework</artifactId>
<version>1.121.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<groupId>com.testframework.lambdas</groupId>
<artifactId>test-framework-lambdas</artifactId>
<description>Lambda - To get build number status</description>
<profiles>
<profile>
<activation>
<os>
<family>!windows</family>
</os>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.0</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<forceJavacCompilerUse>true</forceJavacCompilerUse>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<executions>
<execution>
<!-- Script to Zip lambda files -->
<phase>generate-sources</phase>
<goals>
<goal>exec</goal>
</goals>
<configuration>
<executable>src/main/resources/create-zipfiles-for-lambda.sh</executable>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>

<dependencies>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-lambda-java-events</artifactId>
<version>${aws.lambda.java.events}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-lambda-java-core</artifactId>
<version>${aws-lambda-java-core}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-rds -->
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-rds</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${apache.commons.io.version}</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20180130</version>
</dependency>
<dependency>
<groupId>com.google.collections</groupId>
<artifactId>google-collections</artifactId>
<version>${google-collections}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${com.google.guava}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-lambda</artifactId>
<version>${aws-java-sdk-lambda}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons-logging}</version>
</dependency>
</dependencies>
</project>
***********************************************************************************

Lambdas:

Cloudmongo:

Newrelic:

aws dynamo source:

Cloud watch Events and triggers:

Export Feature :1577 domtoimage, jsPDF

Bar, pie ,line chart: 1586 

python mongo scripts:

Logstash related stuffs:

Success Build Numbers for release and snapshot version for bamboo:











Filezilla use for ec2-user file upload etc,

rm -rf file, pwd --> get folder path, mkdir -p /tmp/es --> mkdir in path /tmp/es, to run any script at backend -->nohup /home/my_user/scripts/my_script.sh >> /home/my_user/scripts/logs/my_script.log 2>&1 & 

nohup python /home/my_user/scripts/myPython.py >> /home/my_user/scripts/logs/my_script.log 2>&1 & 

myscript.sh will run in background and log details in myscript.log

ls -lrt to get list of files and folder with directory,     ./ to run anything in current path ex: ./python.py ./myshell.sh ./multielasticdump if executing from another directory /origin path/multielasticdump 

/home/sys-user$ python mypyhton.py 

/home/sys-user$ sh mypyhton.py 

/home/sys-user$ /opt/script/multilasticdump


Elastic dump and restore from one host of es to another:

python

import os

import time


list_of_indices_to_dumped = ['stg-aws-dynamodb-metrics', 'stg-cloudmongo-opcounter-query', 'stg-cloudmongo-scanandorder']


for index in list_of_indices_to_dumped:

print index

cmd = "./multielasticdump --direction=dump --match='^"+index+".*$' --limit=10000 --input=https://search-test-framework-es7-5r3n4oyw2kddzfmns4oo4okr7m.us-west-2.es.amazonaws.com --output=/tmp/es --ignoreTemplate=true"

print cmd


os.system(cmd)



time.sleep(60)

os.system("./multielasticdump --direction=load --match='^"+index+".*$' --limit=10000 --input=/tmp/es --output=https://vpc-test-framework-es7-vpc-b4flsoliewpodkvnqdyc75dtqa.us-west-2.es.amazonaws.com  --ignoreTemplate=true")

time.sleep(60)


Note: --ignoreTemplate=true can be removed if we want template also to be dumped and restored , in most case we want template also. for some case if template existing is not required use "--ignoreTemplate=true"

to find available space in linux:

df -h

Autoscaling update for Engines:


AmazonAutoScalingClient autoScalingClient = new AmazonAutoScalingClient().withRegion(Regions.US_WEST_2);

String nextToken = null;


DescribeAutoScalingGroupsResult describeResult;

String corexIOELB = buildCorexEngineELBName(TFConstants.IO.getValue());

String corexPROCESSINGELB = buildCorexEngineELBName(TFConstants.PROCESSING.getValue());

String corexDPELB = buildCorexEngineELBName(TFConstants.DP.getValue());

String corexJOBSELB = buildCorexEngineELBName(TFConstants.JOBS.getValue());

boolean isCustom = customScenario.getRunSettings() == RunSettingsPopulator.CUSTOM;

String desiredIOCapacity = isCustom ? customScenario.getCustomIOEngine()

: customScenario.getInboundOutboundEngine();

String desiredPECapacity = isCustom ? customScenario.getCustomProcessEngine()

: customScenario.getProcessEngine();

String desiredDPCapacity = isCustom ? customScenario.getCustomDataPollar()

: customScenario.getDataPollarEngine();

String desiredJOBSCapacity = isCustom ? customScenario.getCustomEngineJobs() : customScenario.getEngineJobs();

DescribeAutoScalingGroupsRequest describeAutoScalingGroupRequest = new DescribeAutoScalingGroupsRequest();

UpdateAutoScalingGroupRequest updateAutoScalinggrouprequest = new UpdateAutoScalingGroupRequest();

do {

describeResult = autoScalingClient

.describeAutoScalingGroups(describeAutoScalingGroupRequest.withNextToken(nextToken));

for (AutoScalingGroup autoscalingGroup : describeResult.getAutoScalingGroups()) {

if (autoscalingGroup.getLoadBalancerNames().contains(corexIOELB)

&& autoscalingGroup.getDesiredCapacity() != 0) {

LOGGER.debug("Autoscaling IO Engine with {}", desiredIOCapacity);

UpdateAutoScalingGroupRequest updatereq = updateAutoScalinggrouprequest

.withAutoScalingGroupName(autoscalingGroup.getAutoScalingGroupName());

updatereq.setMaxSize(Integer.parseInt(desiredIOCapacity));

updatereq.setDesiredCapacity(Integer.parseInt(desiredIOCapacity));

autoScalingClient.updateAutoScalingGroup(updatereq);

LOGGER.debug("Autoscaled IO Engine with {}", desiredIOCapacity);

}

if (autoscalingGroup.getLoadBalancerNames().contains(corexPROCESSINGELB)

&& autoscalingGroup.getDesiredCapacity() != 0) {

LOGGER.debug("Autoscaling Processing Engine with {}", desiredPECapacity);

UpdateAutoScalingGroupRequest updatereq = updateAutoScalinggrouprequest

.withAutoScalingGroupName(autoscalingGroup.getAutoScalingGroupName());

updatereq.setMaxSize(Integer.parseInt(desiredPECapacity));

updatereq.setDesiredCapacity(Integer.parseInt(desiredPECapacity));

autoScalingClient.updateAutoScalingGroup(updatereq);

LOGGER.debug("Autoscaled Processing Engine with {}", desiredPECapacity);

}


if (autoscalingGroup.getLoadBalancerNames().contains(corexDPELB)

&& autoscalingGroup.getDesiredCapacity() != 0) {

LOGGER.debug("Autoscaling DP with {}", desiredDPCapacity);

UpdateAutoScalingGroupRequest updatereq = updateAutoScalinggrouprequest

.withAutoScalingGroupName(autoscalingGroup.getAutoScalingGroupName());

updatereq.setMaxSize(Integer.parseInt(desiredDPCapacity));

updatereq.setDesiredCapacity(Integer.parseInt(desiredDPCapacity));

autoScalingClient.updateAutoScalingGroup(updatereq);

LOGGER.debug("Autoscaled DP with {}", desiredDPCapacity);

}


if (autoscalingGroup.getLoadBalancerNames().contains(corexJOBSELB)

&& autoscalingGroup.getDesiredCapacity() != 0) {

LOGGER.debug("Autoscaling Job Engine with {}", desiredJOBSCapacity);

UpdateAutoScalingGroupRequest updatereq = updateAutoScalinggrouprequest

.withAutoScalingGroupName(autoscalingGroup.getAutoScalingGroupName());

updatereq.setMaxSize(Integer.parseInt(desiredJOBSCapacity));

updatereq.setDesiredCapacity(Integer.parseInt(desiredJOBSCapacity));

autoScalingClient.updateAutoScalingGroup(updatereq);

LOGGER.debug("Autoscaled Job Engine with {}", desiredJOBSCapacity);

}

}

nextToken = describeResult.getNextToken();

} while (nextToken != null);

***********************************************************************************

loadProperties:

static {

Properties prop = loadProperties();


S3_REGION_DEV_STG_DEVINT = Regions.fromName(prop.getProperty("s3RegionOfDevStgDevint"));


TFW_BUILDSTATUS_LAMBDA_REGION = Regions.fromName(prop.getProperty("tfwBuildStatusLambdaRegion"));


RDS_DEV_STG_REGION = Regions.fromName(prop.getProperty("rdsDevStgRegion"));


RDS_DEVINT_REGION = Regions.fromName(prop.getProperty("rdsDevintRegion"));


}

    public static final String PROPERTY_FILE_NAME = "test-framework-api.properties";

private static Properties loadProperties() {
Properties prop = new Properties();
try (InputStream input = MirrorScenarioServiceImpl.class.getClassLoader()
.getResourceAsStream(PROPERTY_FILE_NAME)) {
prop.load(input);
} catch (IOException ex) {
LOGGER.info("Exception when trying to load Properties {}", ex);
}
return prop;
}

  private static Properties loadProperties() {
-        Properties prop = new Properties();
-        try {
-            File file = ResourceUtils.getFile(PROPERTY_FILE_NAME);
-            if (file.exists()) {
-                try (InputStream input = new FileInputStream(file)) {
-                    // load a properties file
-                    prop.load(input);
-                } catch (IOException ex) {
-                    LOGGER.info("Exception when trying to load Properties {}", ex);
-                }
-            }
-        } catch (FileNotFoundException e) {
-            LOGGER.info("exception occured when loading properties {}", e);
-        }
-        return prop;
-    }


 /**

     * This method is used to trigger lambda function.

     * @param mapOfBuild has base path till build number as value, so that it gets the status of that build.  

     * @return map with build number and its appropriate status.

     */

    private static Map<String, String> getBuildStatus(Map<String, String> mapOfBuild) {

        Map<String, Map<String, String>> mapbuilder = Maps.newHashMap();

        mapbuilder.put(TFConstants.BUILD_STATUS.getValue(), mapOfBuild);

        InvokeRequest invokeRequest = new InvokeRequest().withFunctionName(TFConstants.LAMBDA_FUNCTION_NAME.getValue())

                .withPayload(String.valueOf(new Gson().toJson(mapbuilder)));

        AWSLambda awsLambda = AWSLambdaClientBuilder.standard().withRegion(TFW_BUILDSTATUS_LAMBDA_REGION).build();


        InvokeResult invokeResult = awsLambda.invoke(invokeRequest);

        ByteBuffer byteBuffer = invokeResult.getPayload();

        Map<String, String> buildStatus = Maps.newHashMap();

        try {


            String rawJson = new String(byteBuffer.array(), StandardCharsets.UTF_8.name());

            ObjectMapper mapper = new ObjectMapper();

            buildStatus = mapper.readValue(rawJson, Map.class);

        } catch (IOException e) {

            LOGGER.info("Exception Obtained when trying to extract response", e);

        }

        return buildStatus;

    }


Extract RDS Folder:


 @Override

    public List<BuildStatus> extractBuildWithStatus(String path) {

        AmazonS3Client s3Client = new AmazonS3Client().withRegion(S3_REGION_DEV_STG_DEVINT);


        Map<String, String> keyNameList = new HashMap<>();

        List<String> prefixWithBuild = folderExtraction(TFConstants.BASE_S3_BUCKET_NAME.getValue(), s3Client, path);

        prefixWithBuild.parallelStream().forEach(folderwithVersion -> {

            keyNameList.put(folderwithVersion, path + TFConstants.FORWARD_SLASH.getValue() + folderwithVersion);

        });

        List<BuildStatus> BuildStatusDetails = Lists.newArrayList();

        getBuildStatus(keyNameList).entrySet().stream().forEach(buildStatusentry -> {

            BuildStatus buildStatus = new BuildStatus();

            buildStatus.setBuildNumber(String.valueOf(buildStatusentry.getKey()));

            buildStatus.setBuildWithStatus(String.valueOf(buildStatusentry.getValue()));

            BuildStatusDetails.add(buildStatus);

        });

        return BuildStatusDetails;

    }

    

    @Override

    public List<String> extractFolderNames(String path) {

        AmazonS3Client s3Client = new AmazonS3Client().withRegion(S3_REGION_DEV_STG_DEVINT);


        return folderExtraction(TFConstants.BASE_S3_BUCKET_NAME.getValue(), s3Client, path);

    }

    

    private List<String> folderExtraction(String bucketName, AmazonS3 s3client, String prefixToBeFiltered) {

        String prefix = prefixToBeFiltered;

        String delimiter = "/";

        if (!prefix.endsWith(delimiter)) {

            prefix += delimiter;

        }


        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix).withDelimiter(delimiter);

        ObjectListing objects = s3client.listObjects(listObjectsRequest);

        List<String> folderNames = Lists.newArrayList();

        objects.getCommonPrefixes().stream().forEach(action -> {

            String value = action.replaceFirst(Pattern.quote(prefixToBeFiltered), "").replace("/", "");

            folderNames.add(value);

        });

        return folderNames;

    }


    @Override

    public List<String> extractRdsSnapshots(String dbInstanceIdentifier) {

        AmazonRDSClient rds;

        if (dbInstanceIdentifier.contains("devint")) {

            rds = new AmazonRDSClient().withRegion(RDS_DEVINT_REGION);

        } else {

            rds = new AmazonRDSClient().withRegion(RDS_DEV_STG_REGION);

        }

        DescribeDBSnapshotsResult describeDBSnapshotsResult = rds

                .describeDBSnapshots(new DescribeDBSnapshotsRequest().withDBInstanceIdentifier(dbInstanceIdentifier));

        List<String> rdsSnapshots = Lists.newArrayList();

        describeDBSnapshotsResult.getDBSnapshots().stream().forEach(action -> {

            rdsSnapshots.add(action.getDBSnapshotIdentifier());

        });

        return rdsSnapshots;

    }



package com.ghx.testframework.api.uiconfig.pages.action;


import java.util.List;

import java.util.regex.Pattern;


import com.amazonaws.auth.AWSCredentials;

import com.amazonaws.auth.AWSCredentialsProvider;

import com.amazonaws.auth.AWSStaticCredentialsProvider;

import com.amazonaws.auth.BasicAWSCredentials;

import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;

import com.amazonaws.auth.profile.ProfileCredentialsProvider;

import com.amazonaws.regions.Region;

import com.amazonaws.regions.Regions;

import com.amazonaws.services.rds.AmazonRDSClient;

import com.amazonaws.services.rds.model.CopyDBSnapshotRequest;

import com.amazonaws.services.rds.model.DescribeDBClusterSnapshotsRequest;

import com.amazonaws.services.rds.model.DescribeDBClusterSnapshotsResult;

import com.amazonaws.services.rds.model.DescribeDBInstancesRequest;

import com.amazonaws.services.rds.model.DescribeDBInstancesResult;

import com.amazonaws.services.rds.model.DescribeDBSnapshotsRequest;

import com.amazonaws.services.rds.model.DescribeDBSnapshotsResult;

import com.amazonaws.services.rds.model.RestoreDBInstanceFromDBSnapshotRequest;

import com.amazonaws.services.s3.AmazonS3;

import com.amazonaws.services.s3.AmazonS3Client;

import com.amazonaws.services.s3.AmazonS3ClientBuilder;

import com.amazonaws.services.s3.model.ListObjectsRequest;

import com.amazonaws.services.s3.model.ListObjectsV2Request;

import com.amazonaws.services.s3.model.ListObjectsV2Result;

import com.amazonaws.services.s3.model.ObjectListing;

import com.amazonaws.services.s3.model.S3ObjectSummary;

import com.ghx.securemsgstore.exception.MessageStoreException;

import com.ghx.securemsgstore.provider.api.FileOrFolder;

import com.ghx.securemsgstore.provider.api.ProviderType;

import com.ghx.securemsgstore.provider.api.SMSClient;

import com.ghx.securemsgstore.provider.api.SMSConfiguration;

import com.ghx.universal.sms.ConfigSvcSMSConfiguration;



public class AwsTest {


public static void main(String[] args) {

// TODO Auto-generated method stub

System.out.println("hai");


   /* String clientRegion = "US-";

    String bucketName = "ghx-deployment";

    AWSCredentials credentials = new BasicAWSCredentials(

            "AKIAQXL43UGUYIZK4JFY", 

            "RYdsliGRuFjUYjErzVzn1eRyQY0ORgtmuSE5SfB9"

          );

    DefaultAWSCredentialsProviderChain credentials1 = new DefaultAWSCredentialsProviderChain();

    AmazonS3 s3client = AmazonS3ClientBuilder

            .standard()

            .withCredentials(new AWSStaticCredentialsProvider(credentials))

            .withRegion(Regions.US_EAST_1)

            .build();

    

    AmazonS3 s3client1 = AmazonS3ClientBuilder

            .standard()

            .withCredentials(credentials1)

            .withRegion(Regions.US_EAST_1)

            .build();*/

    

    

    

  //  AmazonRDSClient rdsClient = new AmazonRDSClient(/*add your credentials and the proper constructor overload*/);

    //rdsClient.setRegion(Region.getRegion(Regions.US_WEST_2));

    

   // AWSCredentialsProvider.

   // AmazonRDSClient.builder().setCredentials((AWSCredentialsProvider) credentials).; 

    

   // CopyDBSnapshotRequest copySnapshot = new CopyDBSnapshotRequest();

   // copySnapshot.setSourceDBSnapshotIdentifier("rds:egx-tpm-stg-01-2019-04-10-09-13");

    

    AmazonRDSClient rds = new AmazonRDSClient();

    rds.withRegion(Regions.US_WEST_2);

    /*rds.describeDBClusterSnapshots();

    System.out.println("RDS >>>"+rds.describeDBSnapshots().getDBSnapshots().contains("egx-tpm"));

    CopyDBSnapshotRequest copySnapshot = new CopyDBSnapshotRequest();

    copySnapshot.setSourceDBSnapshotIdentifier("egx-tpm-stg-01");

    System.out.println("RDS specific>>>"+copySnapshot);*/

    System.out.println("FFFFFFFFFFFFFF"+rds.describeDBClusters());

    rds.describeDBSnapshots().getDBSnapshots().stream().forEach(action -> System.out.println(action.getDBInstanceIdentifier()));

   System.out.println(">>>>>>>>>>>>>>>>>"+rds.describeDBSnapshots().withMarker("egx-tpm-stg").getDBSnapshots()); 

   

   //System.out.println(rds.describeDBSnapshots(new DescribeDBSnapshotsRequest().withDBInstanceIdentifier("egx-tpm-dev-01")));

   System.out.println(rds.describeDBSnapshots(new DescribeDBSnapshotsRequest().withDBInstanceIdentifier("egx-tpm-stg-01")));


   DescribeDBSnapshotsResult s = rds.describeDBSnapshots(new DescribeDBSnapshotsRequest().withDBInstanceIdentifier("egx-tpm-stg-01"));

   s.getDBSnapshots().stream().forEach(action -> {

   System.out.println("snapshots::::"+action.getDBSnapshotIdentifier());

   });

   /* DescribeDBSnapshotsRequest request = new DescribeDBSnapshotsRequest().withDBSnapshotIdentifier("rds:egx-tpm-stg-01*");

    DescribeDBSnapshotsResult result = rds.describeDBSnapshots(request);

    System.out.println("MMMMMMMMMMM"+result.getDBSnapshots());*/

    

    /*DescribeDBInstancesRequest r = new DescribeDBInstancesRequest().withDBInstanceIdentifier("egx-tpm-stg-01");

    

    System.out.println("VVVVVVVVVVVVVVV"+);*/

//    rds.describeDBSnapshotnew DescribeDBInstancesRequest().withDBInstanceIdentifier("egx-tpm-stg-01")


        //System.out.println("RDS????????????????????"+result.getDBSnapshots());

        

        //    System.out.println("RDS????????????????????"+result.getDBSnapshots());


        /*AmazonS3 s3Client = AmazonS3ClientBuilder.standard()

                .withCredentials(new ProfileCredentialsProvider())

                .withRegion(clientRegion)

                .build();*/

    /*AmazonS3Client s3Client = new AmazonS3Client("your access key", "your secret key", cfg);

    S3DirectoryInfo dir = new S3DirectoryInfo(s3Client, bucketName, "your AmazonS3 folder name");

    foreach (IS3FileSystemInfo file in dir.GetFileSystemInfos())

    {

        Console.WriteLine(file.Name);

        Console.WriteLine(file.Extension);

        Console.WriteLine(file.LastWriteTime);

    }*/

    

  /*  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()

            .withBucketName(bucketName)

            .withPrefix("corex" + "/")

            .withMarker("corex" + "/");


    ObjectListing objectListing = s3client.listObjects(listObjectsRequest);


    for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {

        System.out.println(">>>>>>>>>>"+summary.getKey());

    }*/

    

    

    /*ObjectListing listing = s3client.listObjects("ghx-deployment", "corex/");

System.out.println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>"+listing);

    listing.get

    System.out.println("LISTING"+listingname);

    }*/

    

//folderExtraction(bucketName, s3client,"corex");

//-------------------------------folderExtraction(bucketName, s3client, "corex/1.103.0/develop/4822");

    /*

        System.out.println("Listing objects");


        // maxKeys is set to 2 to demonstrate the use of

        // ListObjectsV2Result.getNextContinuationToken()

        ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucketName).withPrefix("corex/").withMaxKeys(2);

        ListObjectsV2Result result;


        do {

            result = s3client.listObjectsV2(req);


            for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {

                System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize());

            }

            // If there are more than maxKeys keys in the bucket, get a continuation token

            // and list the next objects.

            String token = result.getNextContinuationToken();

            System.out.println("Next Continuation Token: " + token);

            req.setContinuationToken(token);

        } while (result.isTruncated());

    }

*/   

   AmazonS3Client s3Client = new AmazonS3Client().withRegion(Regions.US_EAST_1);

   List<String> s1 =folderExtraction("ghx-deployment", s3Client, "corex");

   System.out.println(">>>>>>>>>>>>>>>>>>>S!!!!!!!!!!!!"+s1);

}


private static List<String> folderExtraction(String bucketName, AmazonS3 s3client,String prefixToBeFiltered) {

String prefix =prefixToBeFiltered;

String delimiter = "/";

if (!prefix.endsWith(delimiter)) {

    prefix += delimiter;

}


ListObjectsRequest listObjectsRequest = new ListObjectsRequest()

        .withBucketName(bucketName).withPrefix(prefix)

        .withDelimiter(delimiter);

ObjectListing objects = s3client.listObjects(listObjectsRequest);

System.out.println("Folders"+objects.getCommonPrefixes());

objects.getCommonPrefixes().stream().forEach(action -> {

int i =action.indexOf(prefixToBeFiltered);

action.replaceFirst(Pattern.quote(prefixToBeFiltered), "");

System.out.println("folder to be given"+action.replaceFirst(Pattern.quote(prefixToBeFiltered), ""));

});

return objects.getCommonPrefixes();

}


}




UI :
broadcast , on, emit, resolve, router, $scope, var,

connect to ec2user:
ssh -i file.pem ec2-user@ipaddress
master admin user:
sudo su


sys-user@SMI-518:~/Softwares/mongodb-linux-x86_64-ubuntu1804-4.0.10-rc0/bin$ ./mongod --dbpath='/home/sys-user/Softwares/data'

git clone ssh://git@stash.ghx.com:7899/egx/all.git

http :>>>>https://stash.ghx.com:7893/scm/egx/all.git

Bit Bucket repositories stored

git clone

mvn clean install  -DskipTests

git checkout branchname

git pull

git commit -m'branchName commit message'

git checkout to origin branch

git pull

git checkout to working branch

git merge develop

git push origin working branch


<div class="row form-group required">
<label for="environment"
       class="col-sm-2 control-label"
       l10n="testFramework.mirror.createScenario.environment"></label>
<div class="col-sm-6">
  <select id="environment"  class="form-control" ng-model="mirrorScenario.targetEnvironment" ng-options="env for env in environments">
    <option value="" l10n="label.select"></option>
    </select>
<div validate-error="environment">
<div class="required"
     l10n="testFramework.mirror.createScenario.environmentIsRequired"></div>
</div>
</div>
</div>


***********************************************************************************


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-dev-5d9f38a37a3e50124c942a4e-testframework-stats_all/_search?pretty=true"


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-dev-5db96476206001102427313b-cpu-utilization/_search?pretty=true"


curl -XPOST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization/_search?v&pretty" -d '
{
"query" : {
"match_all" : {}
}
}'

curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization/_search' -d '
{
"query" : {
"match_all" : {}
}
}'

curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-dev-5db96476206001102427313b-cpu-utilization/_search' -d '
{
"query" : {
"match_all" : {}
}
}'

(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":2,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":26,"max_score":1.0,"hits":[{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW2198sfGb6oRN1s8ZCF","_score":1.0,"_source":{"application":"test-framework-minion","user_p":27.0,"hostname":"10.31.33.232","@timestamp":"2019-10-10T13:58:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21-LhqGb6oRN1s8ZUk","_score":1.0,"_source":{"application":"test-framework-minion","user_p":58.75,"hostname":"10.31.33.232","@timestamp":"2019-10-10T13:59:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21_GQw_4fAvohBONDB","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.499999999999995,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:02:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21_GQw_4fAvohBONDC","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:03:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21_U8gGb6oRN1s8bJq","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.458333333333331,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:04:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21_ySlGb6oRN1s8b4r","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.409836065573771,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:05:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21-o5swEcMd9IvMuHs","_score":1.0,"_source":{"application":"test-framework-minion","user_p":58.75,"hostname":"10.31.33.232","@timestamp":"2019-10-10T13:59:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW21_jnewEcMd9IvMvmu","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:03:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW22AtArwEcMd9IvMxdp","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.368852459016396,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:10:00+00:00"}},{"_index":"testfw-replay-dev-5d9f38a37a3e50124c942a4e-cpu-utilization","_type":"system","_id":"AW22A7r4wEcMd9IvMxx_","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.333333333333326,"hostname":"10.31.33.232","@timestamp":"2019-10-10T14:09:00+00:00"}}]}}


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization_all/_search?pretty=true"


(base) sys-user@SMI-243:~/Corex/all$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization_all/_search?pretty=true"
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 89559,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYuN-_4fAvohBxjy2",
      "_score" : 1.0,
      "_ttl" : 103772,
      "_source" : {
        "application" : "test-framework-gru",
        "user_p" : 0.169491525423767,
        "hostname" : "10.31.34.52",
        "@timestamp" : "2019-10-29T07:18:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYsLgwEcMd9IvwRHv",
      "_score" : 1.0,
      "_ttl" : 95422,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.96721311475399,
        "hostname" : "10.31.36.211",
        "@timestamp" : "2019-10-29T07:18:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYuALGb6oRN1sf0fz",
      "_score" : 1.0,
      "_ttl" : 102889,
      "_source" : {
        "application" : "data-poller",
        "user_p" : 3.99999999999996,
        "hostname" : "10.31.34.252",
        "@timestamp" : "2019-10-29T07:18:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYutqGb6oRN1sf0gm",
      "_score" : 1.0,
      "_ttl" : 105800,
      "_source" : {
        "application" : "ui-invoicing",
        "user_p" : 1.91666666666663,
        "hostname" : "10.31.34.50",
        "@timestamp" : "2019-10-29T07:18:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYgB9wEcMd9IvwQw_",
      "_score" : 1.0,
      "_ttl" : 45659,
      "_source" : {
        "application" : "ui-invoicing",
        "user_p" : 1.66666666666667,
        "hostname" : "10.31.34.50",
        "@timestamp" : "2019-10-29T07:17:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYuLEwEcMd9IvwRJN",
      "_score" : 1.0,
      "_ttl" : 103586,
      "_source" : {
        "application" : "regapi-ws",
        "user_p" : 1.58333333333331,
        "hostname" : "10.31.37.7",
        "@timestamp" : "2019-10-29T07:19:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYXgsGb6oRN1sf0EN",
      "_score" : 1.0,
      "_ttl" : 10762,
      "_source" : {
        "application" : "jobs",
        "user_p" : 13.4745762711865,
        "hostname" : "10.31.37.84",
        "@timestamp" : "2019-10-29T07:15:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WYmM5_4fAvohBxjrT",
      "_score" : 1.0,
      "_ttl" : 70935,
      "_source" : {
        "application" : "jobs",
        "user_p" : 12.0000000000001,
        "hostname" : "10.31.37.84",
        "@timestamp" : "2019-10-29T07:18:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZYOk_4fAvohBxky8",
      "_score" : 1.0,
      "_ttl" : 275842,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 3.68852459016393,
        "hostname" : "10.31.36.211",
        "@timestamp" : "2019-10-29T07:21:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZbrwwEcMd9IvwSQq",
      "_score" : 1.0,
      "_ttl" : 289998,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.39344262295142,
        "hostname" : "10.31.34.191",
        "@timestamp" : "2019-10-29T07:21:00+00:00"
      }
    } ]
  }
}


(base) sys-user@SMI-243:~/Corex/all$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search?pretty=true"
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 89595,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZJjCGb6oRN1sf1OH",
      "_score" : 1.0,
      "_ttl" : -24195,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.83333333333318,
        "hostname" : "10.31.36.211",
        "@timestamp" : "2019-10-29T07:20:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZMFG_4fAvohBxkfH",
      "_score" : 1.0,
      "_ttl" : -13823,
      "_source" : {
        "application" : "ui-invoicing",
        "user_p" : 2.00000000000008,
        "hostname" : "10.31.34.50",
        "@timestamp" : "2019-10-29T07:20:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZcFAwEcMd9IvwSQs",
      "_score" : 1.0,
      "_ttl" : 51707,
      "_source" : {
        "application" : "processing-engine",
        "user_p" : 16.7500000000003,
        "hostname" : "10.31.34.108",
        "@timestamp" : "2019-10-29T07:21:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZYOk_4fAvohBxky8",
      "_score" : 1.0,
      "_ttl" : 35935,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 3.68852459016393,
        "hostname" : "10.31.36.211",
        "@timestamp" : "2019-10-29T07:21:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZbrwwEcMd9IvwSQq",
      "_score" : 1.0,
      "_ttl" : 50091,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.39344262295142,
        "hostname" : "10.31.34.191",
        "@timestamp" : "2019-10-29T07:21:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZNBW_4fAvohBxkgU",
      "_score" : 1.0,
      "_ttl" : -9967,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.58333333333303,
        "hostname" : "10.31.32.198",
        "@timestamp" : "2019-10-29T07:20:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZjAeGb6oRN1sf1_7",
      "_score" : 1.0,
      "_ttl" : 80089,
      "_source" : {
        "application" : "io-engine",
        "user_p" : 24.5833333333333,
        "hostname" : "10.31.59.78",
        "@timestamp" : "2019-10-29T07:21:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZR9tGb6oRN1sf1jF",
      "_score" : 1.0,
      "_ttl" : 10280,
      "_source" : {
        "application" : "ui-mex",
        "user_p" : 5.16666666666652,
        "hostname" : "10.31.34.54",
        "@timestamp" : "2019-10-29T07:20:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZvVZwEcMd9IvwSz9",
      "_score" : 1.0,
      "_ttl" : 130580,
      "_source" : {
        "application" : "ui-mex",
        "user_p" : 26.7499999999999,
        "hostname" : "10.31.34.54",
        "@timestamp" : "2019-10-29T07:22:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4WZg-mwEcMd9IvwSeb",
      "_score" : 1.0,
      "_ttl" : 71777,
      "_source" : {
        "application" : "jobs",
        "user_p" : 12.0833333333333,
        "hostname" : "10.31.37.84",
        "@timestamp" : "2019-10-29T07:22:00+00:00"
      }
    } ]
  }
}


curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization/_search' -d '
{
"query" : {
"match_all" : {}
}
}'


dev-newrelic-cpu-utilization_all
dev-newrelic-cpu-utilization

:https://corex-dev.awsdsi.ghx.com/rest/testframework/performance/metrics/?chartGroup=osdao&chartName=Cumulative+CPU+Utilization&filterType=All&fromDate=2019-10-30T10:24:39.757Z&interval=2m&testResultId=5db96476206001102427313b&toDate=2019-10-30T10:37:04.066Z&viewType=chart
Request method:GET

{"JSON":{"data":[{"10.31.36.192-ui-mex":1.9166666269302368,"10.31.33.225-regapi-ws":1.4754098653793335,"10.31.32.96-ui-heimdall":1.4166666269302368,"10.31.37.146-ui-heimdall":14.916666984558105,"10.31.32.198-ui-heimdall":1.25,"10.31.33.6-jobs":13.389830589294434,"10.31.34.52-test-framework-gru":0.14830508828163147,"10.31.32.27-processing-engine":27.868852615356445,"10.31.36.203-ui-invoicing":1.6666666269302368,"10.31.34.191-ui-heimdall":1.2711864709854126,"10.31.35.26-data-poller":3.852458953857422,"10.31.59.13-io-engine":26.47541046142578,"10.31.33.224-ui-rc":1.5,"time":1572431100000,"10.31.34.100-ui-heimdall":2.7868852615356445},{"10.31.36.192-ui-mex":1.8032786846160889,"10.31.33.225-regapi-ws":1.7796610593795776,"10.31.32.96-ui-heimdall":1.4166666269302368,"10.31.37.146-ui-heimdall":1.7213114500045776,"10.31.32.198-ui-heimdall":1.6949152946472168,"10.31.33.6-jobs":11.803278923034668,"10.31.34.52-test-framework-gru":0.1041666641831398,"10.31.32.27-processing-engine":20.423728942871094,"10.31.36.203-ui-invoicing":2.5833332538604736,"10.31.34.191-ui-heimdall":1.4166666269302368,"10.31.35.26-data-poller":4.237287998199463,"10.31.59.13-io-engine":26,"10.31.33.224-ui-rc":1.8852459192276,"time":1572431160000,"10.31.34.100-ui-heimdall":4.576271057128906},{"10.31.36.192-ui-mex":1.5254237651824951,"10.31.32.96-ui-heimdall":1.3333333730697632,"10.31.33.225-regapi-ws":1.3333333730697632,"10.31.37.146-ui-heimdall":2.6271185874938965,"10.31.35.221-ui-heimdall":68.75,"10.31.37.26-jobs":60.16666793823242,"10.31.33.189-processing-engine":64.91525268554688,"10.31.32.198-ui-heimdall":1.311475396156311,"10.31.59.74-io-engine":62.91666793823242,"10.31.33.78-ui-mex":45.819671630859375,"10.31.35.238-regapi-ws":66.83333587646484,"10.31.33.6-jobs":9.576271057128906,"10.31.34.52-test-framework-gru":0.1666666716337204,"10.31.32.27-processing-engine":17.66666603088379,"10.31.36.203-ui-invoicing":1.6666666269302368,"10.31.37.20-data-poller":66.3559341430664,"10.31.37.102-ui-invoicing":50.08333206176758,"10.31.34.191-ui-heimdall":1.3333333730697632,"10.31.35.107-ui-rc":43.16666793823242,"10.31.35.26-data-poller":4.166666507720947,"10.31.59.13-io-engine":25.3389835357666,"10.31.33.224-ui-rc":1.610169529914856,"time":1572431220000,"10.31.34.100-ui-heimdall":6.666666507720947},{"10.31.36.192-ui-mex":1.5833333730697632,"10.31.32.96-ui-heimdall":1.6666666269302368,"10.31.33.225-regapi-ws":1.5,"10.31.37.146-ui-heimdall":1.5833333730697632,"10.31.35.221-ui-heimdall":72.25,"10.31.37.26-jobs":89.66666412353516,"10.31.33.189-processing-engine":85,"10.31.32.198-ui-heimdall":1.3559322357177734,"10.31.59.74-io-engine":91.80327606201172,"10.31.33.78-ui-mex":85.4237289428711,"10.31.35.238-regapi-ws":75.66666412353516,"10.31.33.6-jobs":11.5,"10.31.34.52-test-framework-gru":0.125,"10.31.32.27-processing-engine":14.25,"10.31.36.203-ui-invoicing":3.1666667461395264,"10.31.37.20-data-poller":79.58333587646484,"10.31.37.102-ui-invoicing":84.75,"10.31.34.191-ui-heimdall":1.3333333730697632,"10.31.35.107-ui-rc":57.5,"10.31.35.26-data-poller":3.7704918384552,"10.31.59.13-io-engine":26.31147575378418,"10.31.33.224-ui-rc":1.5573770999908447,"time":1572431280000,"10.31.34.100-ui-heimdall":6.967213153839111},{"10.31.36.192-ui-mex":1.5,"10.31.33.225-regapi-ws":2.0833332538604736,"10.31.32.96-ui-heimdall":1.1666666269302368,"10.31.37.146-ui-heimdall":1.9166666269302368,"10.31.35.221-ui-heimdall":87.95082092285156,"10.31.37.26-jobs":68.41666412353516,"10.31.33.189-processing-engine":67.91666412353516,"10.31.32.198-ui-heimdall":1.3333333730697632,"10.31.59.74-io-engine":86.08333587646484,"10.31.33.78-ui-mex":80.58333587646484,"10.31.35.238-regapi-ws":89.5,"10.31.33.6-jobs":10.833333015441895,"10.31.34.52-test-framework-gru":0.125,"10.31.32.27-processing-engine":17,"10.31.36.203-ui-invoicing":8.333333015441895,"10.31.37.20-data-poller":68.75,"10.31.37.102-ui-invoicing":78.75,"10.31.34.191-ui-heimdall":1.6393442153930664,"10.31.35.107-ui-rc":87.41666412353516,"10.31.35.26-data-poller":3.8135592937469482,"10.31.59.13-io-engine":25,"10.31.33.224-ui-rc":1.7796610593795776,"time":1572431340000,"10.31.34.100-ui-heimdall":5.677966117858887},{"10.31.36.192-ui-mex":1.9166666269302368,"10.31.32.96-ui-heimdall":1.4166666269302368,"10.31.33.225-regapi-ws":1.4166666269302368,"10.31.37.146-ui-heimdall":2.5833332538604736,"10.31.35.221-ui-heimdall":66.44068145751953,"10.31.37.26-jobs":88.83333587646484,"10.31.33.189-processing-engine":91.91666412353516,"10.31.32.198-ui-heimdall":1.4166666269302368,"10.31.59.74-io-engine":89.40677642822266,"10.31.33.78-ui-mex":65.6557388305664,"10.31.35.238-regapi-ws":85.5,"10.31.33.6-jobs":11.967212677001953,"10.31.34.52-test-framework-gru":0.1458333283662796,"10.31.32.27-processing-engine":30.57377052307129,"10.31.36.203-ui-invoicing":3.25,"10.31.37.20-data-poller":93.75,"10.31.37.102-ui-invoicing":79.08333587646484,"10.31.34.191-ui-heimdall":1.4406780004501343,"10.31.35.107-ui-rc":84.08333587646484,"10.31.35.26-data-poller":3.9166667461395264,"10.31.59.13-io-engine":26.75,"10.31.33.224-ui-rc":1.5833333730697632,"time":1572431400000,"10.31.34.100-ui-heimdall":3.1666667461395264},{"10.31.36.192-ui-mex":1.6393442153930664,"10.31.33.225-regapi-ws":1.9166666269302368,"10.31.32.96-ui-heimdall":1.25,"10.31.37.146-ui-heimdall":1.5,"10.31.35.221-ui-heimdall":82.08333587646484,"10.31.37.26-jobs":84.83333587646484,"10.31.33.189-processing-engine":91,"10.31.32.198-ui-heimdall":1.6666666269302368,"10.31.59.74-io-engine":92.75,"10.31.33.78-ui-mex":67.54237365722656,"10.31.33.6-jobs":9.406779289245605,"10.31.35.238-regapi-ws":6.229508399963379,"10.31.34.52-test-framework-gru":0.1458333283662796,"10.31.32.27-processing-engine":13.389830589294434,"10.31.36.203-ui-invoicing":2,"10.31.37.20-data-poller":89.33333587646484,"10.31.37.102-ui-invoicing":29.8360652923584,"10.31.34.191-ui-heimdall":1.4166666269302368,"10.31.35.107-ui-rc":25.16666603088379,"10.31.35.26-data-poller":3.75,"10.31.59.13-io-engine":24.672130584716797,"10.31.33.224-ui-rc":1.5833333730697632,"time":1572431460000,"10.31.34.100-ui-heimdall":6.166666507720947},{"10.31.36.192-ui-mex":4.06779670715332,"10.31.33.225-regapi-ws":1.5,"10.31.32.96-ui-heimdall":1.3934426307678223,"10.31.37.146-ui-heimdall":1.9166666269302368,"10.31.35.221-ui-heimdall":5.25,"10.31.37.26-jobs":89.25,"10.31.33.189-processing-engine":85.6557388305664,"10.31.32.198-ui-heimdall":1.3333333730697632,"10.31.59.74-io-engine":47,"10.31.33.78-ui-mex":2.25,"10.31.33.6-jobs":13.833333015441895,"10.31.35.238-regapi-ws":2.3728814125061035,"10.31.34.52-test-framework-gru":0.1666666716337204,"10.31.32.27-processing-engine":12,"10.31.36.203-ui-invoicing":2.5,"10.31.37.20-data-poller":62.08333206176758,"10.31.37.102-ui-invoicing":2.4166667461395264,"10.31.34.191-ui-heimdall":1.25,"10.31.35.26-data-poller":3.9166667461395264,"10.31.35.107-ui-rc":2.5,"10.31.59.13-io-engine":24.6610164642334,"10.31.33.224-ui-rc":2.75,"time":1572431520000,"10.31.34.100-ui-heimdall":5.083333492279053},{"10.31.36.192-ui-mex":2.25,"10.31.32.96-ui-heimdall":1.610169529914856,"10.31.33.225-regapi-ws":1.4754098653793335,"10.31.37.146-ui-heimdall":1.6666666269302368,"10.31.35.221-ui-heimdall":8.68852424621582,"10.31.37.26-jobs":89.83606719970703,"10.31.33.189-processing-engine":67.03389739990234,"10.31.32.198-ui-heimdall":1.2295081615447998,"10.31.59.74-io-engine":35.33333206176758,"10.31.33.78-ui-mex":9.083333015441895,"10.31.33.6-jobs":9.583333015441895,"10.31.35.238-regapi-ws":3.0833332538604736,"10.31.34.52-test-framework-gru":0.125,"10.31.37.20-data-poller":18.66666603088379,"10.31.37.102-ui-invoicing":2.8813560009002686,"10.31.34.191-ui-heimdall":1.3333333730697632,"10.31.35.26-data-poller":3.9344263076782227,"10.31.35.107-ui-rc":2.7868852615356445,"10.31.59.13-io-engine":24.25,"10.31.33.224-ui-rc":1.6666666269302368,"time":1572431580000,"10.31.34.100-ui-heimdall":3.524590253829956},{"10.31.33.225-regapi-ws":1.3559322357177734,"10.31.32.96-ui-heimdall":1.3333333730697632,"10.31.37.146-ui-heimdall":1.6666666269302368,"10.31.35.221-ui-heimdall":2.96610164642334,"10.31.37.26-jobs":60.084747314453125,"10.31.33.189-processing-engine":36.33333206176758,"10.31.32.198-ui-heimdall":1.3559322357177734,"10.31.59.74-io-engine":34.41666793823242,"10.31.33.78-ui-mex":1.8333333730697632,"10.31.35.238-regapi-ws":2.0833332538604736,"10.31.34.52-test-framework-gru":0.1458333283662796,"10.31.37.20-data-poller":10.083333015441895,"10.31.37.102-ui-invoicing":1.9166666269302368,"10.31.34.191-ui-heimdall":1.5833333730697632,"10.31.35.107-ui-rc":1.610169529914856,"10.31.59.13-io-engine":24.66666603088379,"time":1572431640000},{"10.31.33.225-regapi-ws":1.5,"10.31.32.96-ui-heimdall":1.3333333730697632,"10.31.37.146-ui-heimdall":2.131147623062134,"10.31.35.221-ui-heimdall":26.66666603088379,"10.31.37.26-jobs":39.5,"10.31.33.189-processing-engine":41.41666793823242,"10.31.32.198-ui-heimdall":1.4166666269302368,"10.31.59.74-io-engine":32.91666793823242,"10.31.33.78-ui-mex":1.6393442153930664,"10.31.35.238-regapi-ws":2.3333332538604736,"10.31.34.52-test-framework-gru":0.125,"10.31.37.20-data-poller":6.833333492279053,"10.31.37.102-ui-invoicing":2.3333332538604736,"10.31.34.191-ui-heimdall":1.3333333730697632,"10.31.35.107-ui-rc":2.1666667461395264,"10.31.59.13-io-engine":26.33333396911621,"time":1572431700000},{"10.31.33.225-regapi-ws":1.5833333730697632,"10.31.32.96-ui-heimdall":1.3333333730697632,"10.31.37.146-ui-heimdall":1.8644068241119385,"10.31.35.221-ui-heimdall":9.916666984558105,"10.31.37.26-jobs":22.83333396911621,"10.31.33.189-processing-engine":21.25,"10.31.32.198-ui-heimdall":1.5833333730697632,"10.31.59.74-io-engine":31.58333396911621,"10.31.33.78-ui-mex":18.983051300048828,"10.31.35.238-regapi-ws":2.6666667461395264,"10.31.34.52-test-framework-gru":0.125,"10.31.37.20-data-poller":5.245901584625244,"10.31.37.102-ui-invoicing":33.25,"10.31.34.191-ui-heimdall":1.4166666269302368,"10.31.35.107-ui-rc":40.33333206176758,"10.31.59.13-io-engine":24.16666603088379,"time":1572431760000},{"10.31.33.225-regapi-ws":1.4754098653793335,"10.31.32.96-ui-heimdall":1.2295081615447998,"10.31.37.146-ui-heimdall":2.25,"10.31.35.221-ui-heimdall":15.916666984558105,"10.31.37.26-jobs":14.583333015441895,"10.31.33.189-processing-engine":28.770492553710938,"10.31.32.198-ui-heimdall":1.3333333730697632,"10.31.59.74-io-engine":28.08333396911621,"10.31.33.78-ui-mex":1.9672131538391113,"10.31.35.238-regapi-ws":2.5409836769104004,"10.31.34.52-test-framework-gru":0.1458333283662796,"10.31.37.20-data-poller":5.338983058929443,"10.31.37.102-ui-invoicing":13.666666984558105,"10.31.34.191-ui-heimdall":1.3333333730697632,"10.31.35.107-ui-rc":3.75,"10.31.59.13-io-engine":23.91666603088379,"time":1572431820000}],"keys":["10.31.32.27-processing-engine","10.31.59.13-io-engine","10.31.37.146-ui-heimdall","10.31.33.6-jobs","10.31.35.26-data-poller","10.31.34.100-ui-heimdall","10.31.36.192-ui-mex","10.31.36.203-ui-invoicing","10.31.33.224-ui-rc","10.31.33.225-regapi-ws","10.31.32.96-ui-heimdall","10.31.34.191-ui-heimdall","10.31.32.198-ui-heimdall","10.31.34.52-test-framework-gru","10.31.35.221-ui-heimdall","10.31.35.238-regapi-ws","10.31.37.20-data-poller","10.31.33.189-processing-engine","10.31.59.74-io-engine","10.31.37.26-jobs","10.31.37.102-ui-invoicing","10.31.33.78-ui-mex","10.31.35.107-ui-rc"],"groupByItems":{"regapi-ws":"regapi-ws","ui-mex":"ui-mex","processing-engine":"processing-engine","jobs":"jobs","ui-invoicing":"ui-invoicing","ui-rc":"ui-rc","data-poller":"data-poller","test-framework-gru":"test-framework-gru","io-engine":"io-engine","ui-heimdall":"ui-heimdall"}},"Response payload":{"EDITOR_CONFIG":{"text":"{\"data\":[{\"10.31.36.192-ui-mex\":1.9166666269302368,\"10.31.33.225-regapi-ws\":1.4754098653793335,\"10.31.32.96-ui-heimdall\":1.4166666269302368,\"10.31.37.146-ui-heimdall\":14.916666984558105,\"10.31.32.198-ui-heimdall\":1.25,\"10.31.33.6-jobs\":13.389830589294434,\"10.31.34.52-test-framework-gru\":0.14830508828163147,\"10.31.32.27-processing-engine\":27.868852615356445,\"10.31.36.203-ui-invoicing\":1.6666666269302368,\"10.31.34.191-ui-heimdall\":1.2711864709854126,\"10.31.35.26-data-poller\":3.852458953857422,\"10.31.59.13-io-engine\":26.47541046142578,\"10.31.33.224-ui-rc\":1.5,\"time\":1572431100000,\"10.31.34.100-ui-heimdall\":2.7868852615356445},{\"10.31.36.192-ui-mex\":1.8032786846160889,\"10.31.33.225-regapi-ws\":1.7796610593795776,\"10.31.32.96-ui-heimdall\":1.4166666269302368,\"10.31.37.146-ui-heimdall\":1.7213114500045776,\"10.31.32.198-ui-heimdall\":1.6949152946472168,\"10.31.33.6-jobs\":11.803278923034668,\"10.31.34.52-test-framework-gru\":0.1041666641831398,\"10.31.32.27-processing-engine\":20.423728942871094,\"10.31.36.203-ui-invoicing\":2.5833332538604736,\"10.31.34.191-ui-heimdall\":1.4166666269302368,\"10.31.35.26-data-poller\":4.237287998199463,\"10.31.59.13-io-engine\":26.0,\"10.31.33.224-ui-rc\":1.8852459192276,\"time\":1572431160000,\"10.31.34.100-ui-heimdall\":4.576271057128906},{\"10.31.36.192-ui-mex\":1.5254237651824951,\"10.31.32.96-ui-heimdall\":1.3333333730697632,\"10.31.33.225-regapi-ws\":1.3333333730697632,\"10.31.37.146-ui-heimdall\":2.6271185874938965,\"10.31.35.221-ui-heimdall\":68.75,\"10.31.37.26-jobs\":60.16666793823242,\"10.31.33.189-processing-engine\":64.91525268554688,\"10.31.32.198-ui-heimdall\":1.311475396156311,\"10.31.59.74-io-engine\":62.91666793823242,\"10.31.33.78-ui-mex\":45.819671630859375,\"10.31.35.238-regapi-ws\":66.83333587646484,\"10.31.33.6-jobs\":9.576271057128906,\"10.31.34.52-test-framework-gru\":0.1666666716337204,\"10.31.32.27-processing-engine\":17.66666603088379,\"10.31.36.203-ui-invoicing\":1.6666666269302368,\"10.31.37.20-data-poller\":66.3559341430664,\"10.31.37.102-ui-invoicing\":50.08333206176758,\"10.31.34.191-ui-heimdall\":1.3333333730697632,\"10.31.35.107-ui-rc\":43.16666793823242,\"10.31.35.26-data-poller\":4.166666507720947,\"10.31.59.13-io-engine\":25.3389835357666,\"10.31.33.224-ui-rc\":1.610169529914856,\"time\":1572431220000,\"10.31.34.100-ui-heimdall\":6.666666507720947},{\"10.31.36.192-ui-mex\":1.5833333730697632,\"10.31.32.96-ui-heimdall\":1.6666666269302368,\"10.31.33.225-regapi-ws\":1.5,\"10.31.37.146-ui-heimdall\":1.5833333730697632,\"10.31.35.221-ui-heimdall\":72.25,\"10.31.37.26-jobs\":89.66666412353516,\"10.31.33.189-processing-engine\":85.0,\"10.31.32.198-ui-heimdall\":1.3559322357177734,\"10.31.59.74-io-engine\":91.80327606201172,\"10.31.33.78-ui-mex\":85.4237289428711,\"10.31.35.238-regapi-ws\":75.66666412353516,\"10.31.33.6-jobs\":11.5,\"10.31.34.52-test-framework-gru\":0.125,\"10.31.32.27-processing-engine\":14.25,\"10.31.36.203-ui-invoicing\":3.1666667461395264,\"10.31.37.20-data-poller\":79.58333587646484,\"10.31.37.102-ui-invoicing\":84.75,\"10.31.34.191-ui-heimdall\":1.3333333730697632,\"10.31.35.107-ui-rc\":57.5,\"10.31.35.26-data-poller\":3.7704918384552,\"10.31.59.13-io-engine\":26.31147575378418,\"10.31.33.224-ui-rc\":1.5573770999908447,\"time\":1572431280000,\"10.31.34.100-ui-heimdall\":6.967213153839111},{\"10.31.36.192-ui-mex\":1.5,\"10.31.33.225-regapi-ws\":2.0833332538604736,\"10.31.32.96-ui-heimdall\":1.1666666269302368,\"10.31.37.146-ui-heimdall\":1.9166666269302368,\"10.31.35.221-ui-heimdall\":87.95082092285156,\"10.31.37.26-jobs\":68.41666412353516,\"10.31.33.189-processing-engine\":67.91666412353516,\"10.31.32.198-ui-heimdall\":1.3333333730697632,\"10.31.59.74-io-engine\":86.08333587646484,\"10.31.33.78-ui-mex\":80.58333587646484,\"10.31.35.238-regapi-ws\":89.5,\"10.31.33.6-jobs\":10.833333015441895,\"10.31.34.52-test-framework-gru\":0.125,\"10.31.32.27-processing-engine\":17.0,\"10.31.36.203-ui-invoicing\":8.333333015441895,\"10.31.37.20-data-poller\":68.75,\"10.31.37.102-ui-invoicing\":78.75,\"10.31.34.191-ui-heimdall\":1.6393442153930664,\"10.31.35.107-ui-rc\":87.41666412353516,\"10.31.35.26-data-poller\":3.8135592937469482,\"10.31.59.13-io-engine\":25.0,\"10.31.33.224-ui-rc\":1.7796610593795776,\"time\":1572431340000,\"10.31.34.100-ui-heimdall\":5.677966117858887},{\"10.31.36.192-ui-mex\":1.9166666269302368,\"10.31.32.96-ui-heimdall\":1.4166666269302368,\"10.31.33.225-regapi-ws\":1.4166666269302368,\"10.31.37.146-ui-heimdall\":2.5833332538604736,\"10.31.35.221-ui-heimdall\":66.44068145751953,\"10.31.37.26-jobs\":88.83333587646484,\"10.31.33.189-processing-engine\":91.91666412353516,\"10.31.32.198-ui-heimdall\":1.4166666269302368,\"10.31.59.74-io-engine\":89.40677642822266,\"10.31.33.78-ui-mex\":65.6557388305664,\"10.31.35.238-regapi-ws\":85.5,\"10.31.33.6-jobs\":11.967212677001953,\"10.31.34.52-test-framework-gru\":0.1458333283662796,\"10.31.32.27-processing-engine\":30.57377052307129,\"10.31.36.203-ui-invoicing\":3.25,\"10.31.37.20-data-poller\":93.75,\"10.31.37.102-ui-invoicing\":79.08333587646484,\"10.31.34.191-ui-heimdall\":1.4406780004501343,\"10.31.35.107-ui-rc\":84.08333587646484,\"10.31.35.26-data-poller\":3.9166667461395264,\"10.31.59.13-io-engine\":26.75,\"10.31.33.224-ui-rc\":1.5833333730697632,\"time\":1572431400000,\"10.31.34.100-ui-heimdall\":3.1666667461395264},{\"10.31.36.192-ui-mex\":1.6393442153930664,\"10.31.33.225-regapi-ws\":1.9166666269302368,\"10.31.32.96-ui-heimdall\":1.25,\"10.31.37.146-ui-heimdall\":1.5,\"10.31.35.221-ui-heimdall\":82.08333587646484,\"10.31.37.26-jobs\":84.83333587646484,\"10.31.33.189-processing-engine\":91.0,\"10.31.32.198-ui-heimdall\":1.6666666269302368,\"10.31.59.74-io-engine\":92.75,\"10.31.33.78-ui-mex\":67.54237365722656,\"10.31.33.6-jobs\":9.406779289245605,\"10.31.35.238-regapi-ws\":6.229508399963379,\"10.31.34.52-test-framework-gru\":0.1458333283662796,\"10.31.32.27-processing-engine\":13.389830589294434,\"10.31.36.203-ui-invoicing\":2.0,\"10.31.37.20-data-poller\":89.33333587646484,\"10.31.37.102-ui-invoicing\":29.8360652923584,\"10.31.34.191-ui-heimdall\":1.4166666269302368,\"10.31.35.107-ui-rc\":25.16666603088379,\"10.31.35.26-data-poller\":3.75,\"10.31.59.13-io-engine\":24.672130584716797,\"10.31.33.224-ui-rc\":1.5833333730697632,\"time\":1572431460000,\"10.31.34.100-ui-heimdall\":6.166666507720947},{\"10.31.36.192-ui-mex\":4.06779670715332,\"10.31.33.225-regapi-ws\":1.5,\"10.31.32.96-ui-heimdall\":1.3934426307678223,\"10.31.37.146-ui-heimdall\":1.9166666269302368,\"10.31.35.221-ui-heimdall\":5.25,\"10.31.37.26-jobs\":89.25,\"10.31.33.189-processing-engine\":85.6557388305664,\"10.31.32.198-ui-heimdall\":1.3333333730697632,\"10.31.59.74-io-engine\":47.0,\"10.31.33.78-ui-mex\":2.25,\"10.31.33.6-jobs\":13.833333015441895,\"10.31.35.238-regapi-ws\":2.3728814125061035,\"10.31.34.52-test-framework-gru\":0.1666666716337204,\"10.31.32.27-processing-engine\":12.0,\"10.31.36.203-ui-invoicing\":2.5,\"10.31.37.20-data-poller\":62.08333206176758,\"10.31.37.102-ui-invoicing\":2.4166667461395264,\"10.31.34.191-ui-heimdall\":1.25,\"10.31.35.26-data-poller\":3.9166667461395264,\"10.31.35.107-ui-rc\":2.5,\"10.31.59.13-io-engine\":24.6610164642334,\"10.31.33.224-ui-rc\":2.75,\"time\":1572431520000,\"10.31.34.100-ui-heimdall\":5.083333492279053},{\"10.31.36.192-ui-mex\":2.25,\"10.31.32.96-ui-heimdall\":1.610169529914856,\"10.31.33.225-regapi-ws\":1.4754098653793335,\"10.31.37.146-ui-heimdall\":1.6666666269302368,\"10.31.35.221-ui-heimdall\":8.68852424621582,\"10.31.37.26-jobs\":89.83606719970703,\"10.31.33.189-processing-engine\":67.03389739990234,\"10.31.32.198-ui-heimdall\":1.2295081615447998,\"10.31.59.74-io-engine\":35.33333206176758,\"10.31.33.78-ui-mex\":9.083333015441895,\"10.31.33.6-jobs\":9.583333015441895,\"10.31.35.238-regapi-ws\":3.0833332538604736,\"10.31.34.52-test-framework-gru\":0.125,\"10.31.37.20-data-poller\":18.66666603088379,\"10.31.37.102-ui-invoicing\":2.8813560009002686,\"10.31.34.191-ui-heimdall\":1.3333333730697632,\"10.31.35.26-data-poller\":3.9344263076782227,\"10.31.35.107-ui-rc\":2.7868852615356445,\"10.31.59.13-io-engine\":24.25,\"10.31.33.224-ui-rc\":1.6666666269302368,\"time\":1572431580000,\"10.31.34.100-ui-heimdall\":3.524590253829956},{\"10.31.33.225-regapi-ws\":1.3559322357177734,\"10.31.32.96-ui-heimdall\":1.3333333730697632,\"10.31.37.146-ui-heimdall\":1.6666666269302368,\"10.31.35.221-ui-heimdall\":2.96610164642334,\"10.31.37.26-jobs\":60.084747314453125,\"10.31.33.189-processing-engine\":36.33333206176758,\"10.31.32.198-ui-heimdall\":1.3559322357177734,\"10.31.59.74-io-engine\":34.41666793823242,\"10.31.33.78-ui-mex\":1.8333333730697632,\"10.31.35.238-regapi-ws\":2.0833332538604736,\"10.31.34.52-test-framework-gru\":0.1458333283662796,\"10.31.37.20-data-poller\":10.083333015441895,\"10.31.37.102-ui-invoicing\":1.9166666269302368,\"10.31.34.191-ui-heimdall\":1.5833333730697632,\"10.31.35.107-ui-rc\":1.610169529914856,\"10.31.59.13-io-engine\":24.66666603088379,\"time\":1572431640000},{\"10.31.33.225-regapi-ws\":1.5,\"10.31.32.96-ui-heimdall\":1.3333333730697632,\"10.31.37.146-ui-heimdall\":2.131147623062134,\"10.31.35.221-ui-heimdall\":26.66666603088379,\"10.31.37.26-jobs\":39.5,\"10.31.33.189-processing-engine\":41.41666793823242,\"10.31.32.198-ui-heimdall\":1.4166666269302368,\"10.31.59.74-io-engine\":32.91666793823242,\"10.31.33.78-ui-mex\":1.6393442153930664,\"10.31.35.238-regapi-ws\":2.3333332538604736,\"10.31.34.52-test-framework-gru\":0.125,\"10.31.37.20-data-poller\":6.833333492279053,\"10.31.37.102-ui-invoicing\":2.3333332538604736,\"10.31.34.191-ui-heimdall\":1.3333333730697632,\"10.31.35.107-ui-rc\":2.1666667461395264,\"10.31.59.13-io-engine\":26.33333396911621,\"time\":1572431700000},{\"10.31.33.225-regapi-ws\":1.5833333730697632,\"10.31.32.96-ui-heimdall\":1.3333333730697632,\"10.31.37.146-ui-heimdall\":1.8644068241119385,\"10.31.35.221-ui-heimdall\":9.916666984558105,\"10.31.37.26-jobs\":22.83333396911621,\"10.31.33.189-processing-engine\":21.25,\"10.31.32.198-ui-heimdall\":1.5833333730697632,\"10.31.59.74-io-engine\":31.58333396911621,\"10.31.33.78-ui-mex\":18.983051300048828,\"10.31.35.238-regapi-ws\":2.6666667461395264,\"10.31.34.52-test-framework-gru\":0.125,\"10.31.37.20-data-poller\":5.245901584625244,\"10.31.37.102-ui-invoicing\":33.25,\"10.31.34.191-ui-heimdall\":1.4166666269302368,\"10.31.35.107-ui-rc\":40.33333206176758,\"10.31.59.13-io-engine\":24.16666603088379,\"time\":1572431760000},{\"10.31.33.225-regapi-ws\":1.4754098653793335,\"10.31.32.96-ui-heimdall\":1.2295081615447998,\"10.31.37.146-ui-heimdall\":2.25,\"10.31.35.221-ui-heimdall\":15.916666984558105,\"10.31.37.26-jobs\":14.583333015441895,\"10.31.33.189-processing-engine\":28.770492553710938,\"10.31.32.198-ui-heimdall\":1.3333333730697632,\"10.31.59.74-io-engine\":28.08333396911621,\"10.31.33.78-ui-mex\":1.9672131538391113,\"10.31.35.238-regapi-ws\":2.5409836769104004,\"10.31.34.52-test-framework-gru\":0.1458333283662796,\"10.31.37.20-data-poller\":5.338983058929443,\"10.31.37.102-ui-invoicing\":13.666666984558105,\"10.31.34.191-ui-heimdall\":1.3333333730697632,\"10.31.35.107-ui-rc\":3.75,\"10.31.59.13-io-engine\":23.91666603088379,\"time\":1572431820000}],\"keys\":[\"10.31.32.27-processing-engine\",\"10.31.59.13-io-engine\",\"10.31.37.146-ui-heimdall\",\"10.31.33.6-jobs\",\"10.31.35.26-data-poller\",\"10.31.34.100-ui-heimdall\",\"10.31.36.192-ui-mex\",\"10.31.36.203-ui-invoicing\",\"10.31.33.224-ui-rc\",\"10.31.33.225-regapi-ws\",\"10.31.32.96-ui-heimdall\",\"10.31.34.191-ui-heimdall\",\"10.31.32.198-ui-heimdall\",\"10.31.34.52-test-framework-gru\",\"10.31.35.221-ui-heimdall\",\"10.31.35.238-regapi-ws\",\"10.31.37.20-data-poller\",\"10.31.33.189-processing-engine\",\"10.31.59.74-io-engine\",\"10.31.37.26-jobs\",\"10.31.37.102-ui-invoicing\",\"10.31.33.78-ui-mex\",\"10.31.35.107-ui-rc\"],\"groupByItems\":{\"regapi-ws\":\"regapi-ws\",\"ui-mex\":\"ui-mex\",\"processing-engine\":\"processing-engine\",\"jobs\":\"jobs\",\"ui-invoicing\":\"ui-invoicing\",\"ui-rc\":\"ui-rc\",\"data-poller\":\"data-poller\",\"test-framework-gru\":\"test-framework-gru\",\"io-engine\":\"io-engine\",\"ui-heimdall\":\"ui-heimdall\"}}","mode":"application/json"}}}


(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":161,"max_score":1.0,"hits":[{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCC","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-12T10:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCI","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-13T04:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCN","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-13T19:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCP","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-14T01:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCR","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-14T07:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCV","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-14T19:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCW","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-14T22:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCZ","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-15T07:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCn","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-17T01:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCo","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-17T04:27:00+00:00"}}]}}


dev-newrelic-cpu-utilization_all

dev-newrelic-cpu-utilization

curl -X GET "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization/_mapping?pretty"

ys-user@SMI-243:~/Corex/all$ curl -X GET "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization/_mapping?pretty"
{
  "testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization" : {
    "mappings" : {
      "system" : {
        "properties" : {
          "@timestamp" : {
            "type" : "date",
            "format" : "strict_date_optional_time||epoch_millis"
          },
          "application" : {
            "type" : "string"
          },
          "hostname" : {
            "type" : "string"
          },
          "user_p" : {
            "type" : "long"
          }
        }
      }
    }
  }
}


curl -X GET "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization/_settings?pretty"

(base) sys-user@SMI-243:~/Corex/all$ curl -X GET "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization/_settings?pretty"
{
  "testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization" : {
    "settings" : {
      "index" : {
        "creation_date" : "1572585869611",
        "number_of_shards" : "5",
        "number_of_replicas" : "1",
        "uuid" : "sBhkfNoCTQKmX-kgTTKcow",
        "version" : {
          "created" : "2030299"
        }
      }
    }
  }
}

curl -X PUT "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'{"settings": {"index": {"number_of_shards": 1,"number_of_replicas": 1}},"mappings" : {"system" : {"properties" : {"@timestamp" : {"type" : "date","format" : "strict_date_optional_time||epoch_millis"},"application" : {"type" : "string"},"hostname" : {"type" : "string"},"user_p" : {"type" : "long"}}}},"aliases": {"dev-newrelic-cpu-utilization_all": {}}}'

curl -X PUT "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'{"settings": {"index": {"number_of_shards": 1,"number_of_replicas": 1}},   "mappings" : {"system" : {"_ttl" : {"enabled" : true,"default" : 259200000},"properties" : {"@timestamp" : {"type" : "date","format" : "strict_date_optional_time||epoch_millis"},"application" : {"type" : "string","analyzer" : "keyword"},"hostname" : {"type" : "string","analyzer" : "keyword"},"user_p" : {"type" : "float"}}}},"aliases": {"dev-newrelic-cpu-utilization_all": {}}}'


{
  "dev-cpu-utilization" : {
    "mappings" : {
      "system" : {
        "_ttl" : {
          "enabled" : true,
          "default" : 259200000
        },
        "properties" : {
          "@timestamp" : {
            "type" : "date",
            "format" : "strict_date_optional_time||epoch_millis"
          },
          "application" : {
            "type" : "string",
            "analyzer" : "keyword"
          },
          "hostname" : {
            "type" : "string",
            "analyzer" : "keyword"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      }
    }
  }
}





"mappings": {"metadata": {"_source": {"enabled": true},"properties": {"id": {"type": "long"},"user_p": {"type": "float"},"hostIp": {"analyzer": "custom_keyword","type": "string"},"hostname": {"analyzer": "custom_keyword","type": "string"},"application": {"analyzer": "custom_keyword","type": "string"}}}},"aliases": {"dev-newrelic-cpu-utilization_all": {}}}'


curl -X PUT "localhost:9200/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'{"settings": {"index" : {"creation_date" : "1572585869611","number_of_shards" : "5","number_of_replicas" : "1",
        "uuid" : "sBhkfNoCTQKmX-kgTTKcow",
        "version" : {
          "created" : "2030299"
        }
      }
    }

curl -X GET "localhost:9200/dev-newrelic-cpu-utilization/_search?pretty=true"

curl -X GET "localhost:9200/dev-newrelic-cpu-utilization/_settings?pretty"
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_settings?pretty"

(base) sys-user@SMI-243:~/Corex/all$ curl -X GET "localhost:9200/dev-newrelic-cpu-utilization/_settings?pretty"
{
  "dev-newrelic-cpu-utilization" : {
    "settings" : {
      "index" : {
        "number_of_shards" : "1",
        "provided_name" : "dev-newrelic-cpu-utilization",
        "creation_date" : "1572440094821",
        "analysis" : {
          "analyzer" : {
            "custom_keyword" : {
              "filter" : "lowercase",
              "type" : "custom",
              "tokenizer" : "keyword"
            }
          }
        },
        "number_of_replicas" : "1",
        "uuid" : "KOl5cRUoS9GN3XXPu_VjPA",
        "version" : {
          "created" : "5060899"
        }
      }
    }
  }
}


ys-user@SMI-243:~/Corex/all$ curl -X GET "localhost:9200/dev-newrelic-cpu-utilization/_mapping?pretty"l

(base) sys-user@SMI-243:~/Corex/all$ curl -X GET "localhost:9200/dev-newrelic-cpu-utilization/_mapping?pretty"
{
  "dev-newrelic-cpu-utilization" : {
    "mappings" : {
      "metadata" : {
        "properties" : {
          "application" : {
            "type" : "text",
            "analyzer" : "custom_keyword"
          },
          "hostIp" : {
            "type" : "text",
            "analyzer" : "custom_keyword"
          },
          "hostname" : {
            "type" : "text",
            "analyzer" : "custom_keyword"
          },
          "id" : {
            "type" : "long"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      },
      "system" : {
        "properties" : {
          "@timestamp" : {
            "type" : "date"
          },
          "application" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            },
            "analyzer" : "custom_keyword"
          },
          "hostIp" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            },
            "analyzer" : "custom_keyword"
          },
          "hostname" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            },
            "analyzer" : "custom_keyword"
          },
          "id" : {
            "type" : "long"
          },
          "percent" : {
            "type" : "float"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      }
    }
  }
}


curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization_all/_search' -d '
{
    "query" : {
        "match_all" : {}
    }
}'

curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search' -d '
{
    "query" : {
        "match_all" : {}
    }
}'


> {

https://internal-testfw-cxtf-1405-ui-heimdall-elb-30459250.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/?chartGroup=osdao&chartName=Cumulative+CPU+Utilization&filterType=All&fromDate=2019-11-01T07:44:12.039Z&interval=2m&testResultId=5dbbe1e83109af1032a26531&toDate=2019-11-01T07:56:07.429Z&viewType=chart


https://internal-testfw-cxtf-1405-ui-heimdall-elb-30459250.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/newrelic?chartGroup=osdao&chartName=Cumulative+CPU+Utilization&filterType=All&fromDate=2019-11-01T07:44:12.039Z&interval=2m&testResultId=5dbbe1e83109af1032a26531&toDate=2019-11-01T07:56:07.429Z&viewType=chart
10:58:00+00:00

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"DEV-data-poller","user_p":0.666666666666664,"hostname":"10.31.33.240","@timestamp":"2019-11-01T07:45:12:00+00:00"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"test-framework-minion","user_p":0.466666666666664,"hostname":"10.31.33.232","@timestamp":"2019-10-30T07:45:12.039Z"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"test-framework-minion","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-11-01T07:50:12.039Z"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"test-framework-minion","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-11-01T07:45:12.039Z"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"test-framework-minion","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-11-01T07:45:12.039Z"}'

{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-11-01T07:45:12.039Z"}


{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCC","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-12T10:27:00+00:00"}},{"_index":"testfw-replay-newelic-dev-5dbac3817e0efc1022fb1529-cpu-utilization","_type":"system","_id":"AW4lbC56wEcMd9IvKLCI","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-10-13T04:27:00+00:00"}}


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/_cat/indices?v&pretty"


dev-cpu-utilization


curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search' -d '
{
"query" : {
"match_all" : {}
}
}'


curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search' -d '
{
"query" : {
"match_all" : {}
}
}'


(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization_all/_search' -d '
{
"query" : {
"match_all" : {}
}
}'
{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":1.0,"hits":[{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mjY91Gb6oRN1s7_Y1","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-11-01T07:45:12.039Z"}},{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mlM7c_4fAvohBNuHD","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-11-01T07:45:12.039Z"}},{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mpGaRwEcMd9IvMimc","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-11-01T07:45:12.039Z"}},{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mpOP7_4fAvohBN2BX","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-11-01T07:50:12.039Z"}}]}}


(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":91428,"max_score":1.0,"hits":[{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLKsz_4fAvohBy47t","_score":1.0,"_ttl":5890,"_source":{"application":"test-framework-gru","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLNFqGb6oRN1shJFt","_score":1.0,"_ttl":15673,"_source":{"application":"processing-engine","user_p":13.4166666666666,"hostname":"10.31.32.6","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLMBXGb6oRN1shJDj","_score":1.0,"_ttl":11302,"_source":{"application":"ui-heimdall","user_p":1.41666666666576,"hostname":"10.31.34.191","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLAgZwEcMd9IvxmZy","_score":1.0,"_ttl":-35864,"_source":{"application":"ui-heimdall","user_p":11.4166666666667,"hostname":"10.31.34.111","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLLOr_4fAvohBy4-F","_score":1.0,"_ttl":8058,"_source":{"application":"ui-heimdall","user_p":1.50000000000006,"hostname":"10.31.32.96","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLb_m_4fAvohBy5bU","_score":1.0,"_ttl":76725,"_source":{"application":"regapi-ws","user_p":1.58333333333331,"hostname":"10.31.33.231","@timestamp":"2019-10-29T10:59:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLUdK_4fAvohBy5Ir","_score":1.0,"_ttl":45849,"_source":{"application":"io-engine","user_p":25.4999999999999,"hostname":"10.31.60.142","@timestamp":"2019-10-29T11:00:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLQn5wEcMd9Ivxmwr","_score":1.0,"_ttl":30152,"_source":{"application":"jobs","user_p":14.0983606557376,"hostname":"10.31.36.86","@timestamp":"2019-10-29T10:57:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLfTzwEcMd9IvxnI_","_score":1.0,"_ttl":90306,"_source":{"application":"jobs","user_p":14.4999999999999,"hostname":"10.31.36.86","@timestamp":"2019-10-29T11:00:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XL2wLGb6oRN1shKH9","_score":1.0,"_ttl":186330,"_source":{"application":"test-framework-gru","user_p":0.166666666666705,"hostname":"10.31.34.52","@timestamp":"2019-10-29T11:01:00+00:00"}}]}}(base)

(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":1,"max_score":1.0,"hits":[{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mjY91Gb6oRN1s7_Y1","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-11-01T07:45:12.039Z"}}]}}(


(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":1,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":91428,"max_score":1.0,"hits":[{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLKsz_4fAvohBy47t","_score":1.0,"_ttl":5890,"_source":{"application":"test-framework-gru","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLNFqGb6oRN1shJFt","_score":1.0,"_ttl":15673,"_source":{"application":"processing-engine","user_p":13.4166666666666,"hostname":"10.31.32.6","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLMBXGb6oRN1shJDj","_score":1.0,"_ttl":11302,"_source":{"application":"ui-heimdall","user_p":1.41666666666576,"hostname":"10.31.34.191","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLAgZwEcMd9IvxmZy","_score":1.0,"_ttl":-35864,"_source":{"application":"ui-heimdall","user_p":11.4166666666667,"hostname":"10.31.34.111","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLLOr_4fAvohBy4-F","_score":1.0,"_ttl":8058,"_source":{"application":"ui-heimdall","user_p":1.50000000000006,"hostname":"10.31.32.96","@timestamp":"2019-10-29T10:58:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLb_m_4fAvohBy5bU","_score":1.0,"_ttl":76725,"_source":{"application":"regapi-ws","user_p":1.58333333333331,"hostname":"10.31.33.231","@timestamp":"2019-10-29T10:59:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLUdK_4fAvohBy5Ir","_score":1.0,"_ttl":45849,"_source":{"application":"io-engine","user_p":25.4999999999999,"hostname":"10.31.60.142","@timestamp":"2019-10-29T11:00:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLQn5wEcMd9Ivxmwr","_score":1.0,"_ttl":30152,"_source":{"application":"jobs","user_p":14.0983606557376,"hostname":"10.31.36.86","@timestamp":"2019-10-29T10:57:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XLfTzwEcMd9IvxnI_","_score":1.0,"_ttl":90306,"_source":{"application":"jobs","user_p":14.4999999999999,"hostname":"10.31.36.86","@timestamp":"2019-10-29T11:00:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XL2wLGb6oRN1shKH9","_score":1.0,"_ttl":186330,"_source":{"application":"test-framework-gru","user_p":0.166666666666705,"hostname":"10.31.34.52","@timestamp":"2019-10-29T11:01:00+00:00"}}]}}(




curl -X PUT "localhost:9200/my_index?pretty" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "my_type": {
      "properties": {
        "my_number": {
          "type": "long",
          "fields": {
            "keyword": {
              "type":  "keyword"
            }
          }
        }
      }
    }
  }
}
'

curl -X PUT "localhost:9200/twitter?pretty" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "tweet": {
      "properties": {
        "message": {
          "type": "text"
        }
      }
    }
  }
}
'
curl -X PUT "localhost:9200/twitter/_mapping/user?pretty" -H 'Content-Type: application/json' -d'
{
  "properties": {
    "name": {
      "type": "text"
    }
  }
}
'
curl -X PUT "localhost:9200/twitter/_mapping/tweet?pretty" -H 'Content-Type: application/json' -d'
{
  "properties": {
    "user_name": {
      "type": "text"
    }
  }
}
'
curl -X PUT "localhost:9200/my_index?pretty" -H 'Content-Type: application/json' -d'
{
  "mappings": {
    "user": {
      "properties": {
        "name": {
          "properties": {
            "first": {
              "type": "text"
            }
          }
        },
        "user_id": {
          "type": "keyword"
        }
      }
    }
  }
}
'
curl -X PUT "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_mapping/system?pretty" -H 'Content-Type: application/json' -d'
{
  "properties": {
    "name": {
      "properties": {
        "last": { 
          "type": "text"
        }
      }
    },
    "user_id": {
      "type": "keyword",
      "ignore_above": 100 
    }
  }
}
'

curl -X PUT "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'
{"mappings" :{"system" : {"_ttl": {"default": "3d","enabled": true},{"properties" : {"@timestamp" : {
"type" : "date","format" : "strict_date_optional_time||epoch_millis"
},"application" : {"type" : "string"},"hostname" : {"type" : "string"},"user_p" : {"type" : "long"}}}}}
}
'



(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization_all/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":1,"timed_out":false,"_shards":{"total":1,"successful":1,"failed":0},"hits":{"total":4,"max_score":1.0,"hits":[{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mjY91Gb6oRN1s7_Y1","_score":1.0,"_source":{"application":"DEV-data-poller","user_p":0,"hostname":"10.31.33.240","@timestamp":"2019-11-01T07:45:12.039Z"}},{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mlM7c_4fAvohBNuHD","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-11-01T07:45:12.039Z"}},{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mpGaRwEcMd9IvMimc","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-11-01T07:45:12.039Z"}},{"_index":"dev-newrelic-cpu-utilization","_type":"system","_id":"AW4mpOP7_4fAvohBN2BX","_score":1.0,"_source":{"application":"test-framework-minion","user_p":0.187499999999924,"hostname":"10.31.34.52","@timestamp":"2019-11-01T07:50:12.039Z"}}]}}

curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization_all/_search' -d '
{
"query" : {
"match_all" : {}
}
}'



(base) sys-user@SMI-243:~/Corex/all$ curl -XPOST 'search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization_all/_search' -d '
> {
> "query" : {
> "match_all" : {}
> }
> }'
{"took":2,"timed_out":false,"_shards":{"total":5,"successful":5,"failed":0},"hits":{"total":91779,"max_score":1.0,"hits":[{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUWAgGb6oRN1shX5C","_score":1.0,"_ttl":6991,"_source":{"application":"test-framework-gru","user_p":0.166666666666705,"hostname":"10.31.34.52","@timestamp":"2019-10-29T11:38:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUXcr_4fAvohBzIN6","_score":1.0,"_ttl":12890,"_source":{"application":"ui-heimdall","user_p":1.61016949152512,"hostname":"10.31.34.191","@timestamp":"2019-10-29T11:38:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUYO9wEcMd9Ivx1fa","_score":1.0,"_ttl":16108,"_source":{"application":"ui-heimdall","user_p":1.58333333333455,"hostname":"10.31.32.198","@timestamp":"2019-10-29T11:38:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUWqZGb6oRN1shX53","_score":1.0,"_ttl":9672,"_source":{"application":"ui-heimdall","user_p":1.39344262295076,"hostname":"10.31.32.96","@timestamp":"2019-10-29T11:39:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUPGG_4fAvohBzH61","_score":1.0,"_ttl":-21323,"_source":{"application":"data-poller","user_p":3.99999999999996,"hostname":"10.31.33.9","@timestamp":"2019-10-29T11:38:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUktXGb6oRN1shYM7","_score":1.0,"_ttl":67206,"_source":{"application":"test-framework-gru","user_p":0.125000000000076,"hostname":"10.31.34.52","@timestamp":"2019-10-29T11:39:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUpOhGb6oRN1shYSL","_score":1.0,"_ttl":85712,"_source":{"application":"ui-heimdall","user_p":4.23728813559322,"hostname":"10.31.34.111","@timestamp":"2019-10-29T11:40:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUlWEwEcMd9Ivx11q","_score":1.0,"_ttl":69811,"_source":{"application":"ui-heimdall","user_p":1.44067796610173,"hostname":"10.31.32.96","@timestamp":"2019-10-29T11:40:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUgHxwEcMd9Ivx1nW","_score":1.0,"_ttl":48416,"_source":{"application":"io-engine","user_p":24.7457627118643,"hostname":"10.31.60.142","@timestamp":"2019-10-29T11:40:00+00:00"}},{"_index":"dev-cpu-utilization","_type":"system","_id":"AW4XUseFGb6oRN1shYTf","_score":1.0,"_ttl":98996,"_source":{"application":"data-poller","user_p":4.16666666666667,"hostname":"10.31.33.9","@timestamp":"2019-10-29T11:40:00+00:00"}}]}}

{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":" 2019-10-29T06:59:000Z ","lte":"2019-11-01T07:45:12.040Z" }}},{"term":{"application":"All"}},"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,"aggs":{"output":{"date_histogram":{"field":"@timestamp","interval":1,"time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-30T07:44:12.039Z","max":"2019-11-01T07:56:07.429Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"user_p"}}}}}}}
 


{\"bool\":{\"must\":{\"match_all\":{}},\"filter\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"gte\":" 2019-10-29T06:59:000Z ",\"lte\":" + "2019-11-01T07:45:12.040Z" }}}");
,{\"term\":{\"application\":\"" + filterType + "\"}}

,\"must_not\":[{ \"range\": { \"user_p\": { \"lte\": 0 } } }]}}}}

\"size\":0,\"aggs\":{\"output\":{\"date_histogram\":{\"field\":\"@timestamp\",\"interval\":"
                            + qInterval
                            + ",\"time_zone\":\"+05:30\",\"min_doc_count\":1,\"extended_bounds\":{\"min\":"
                            + qFromDate
                            + ",\"max\":"
                            + qToDate
                            + "}},\"aggs\":{\"items\":{\"terms\":{\"field\":\"hostname\",\"size\":100,\"order\":{\"count\":\"desc\"}},\"aggs\":{\"count\":{\"avg\":{\"field\":\"user_p\"}}}}}}}";
 



curl -X DELETE "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty"
stg-newrelic-heap-utilization

curl -X DELETE "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization?pretty"

curl -X PUT "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'{"settings": {"index": {"number_of_shards": 1,"number_of_replicas": 1}},"mappings" : {"system" : {"properties" : "application" : {"type" : "string"},"hostname" : {"type" : "string"},"user_p" : {"type" : "float"}}}},"aliases": {"dev-newrelic-cpu-utilization_all": {}}}'

curl -X PUT "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization?pretty" -H 'Content-Type: application/json' -d'{"settings": {"index": {"number_of_shards": 1,"number_of_replicas": 1}},"mappings" : {"system" : {"properties" : "application" : {"type" : "string"},"hostname" : {"type" : "string"},"user_p" : {"type" : "float"}}}},"aliases": {"dev-newrelic-cpu-utilization_all": {}}}'


(base) sys-user@SMI-243:~/backup/aws$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty=true"
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 100602,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsU5hwEcMd9IvKq7M",
      "_score" : 1.0,
      "_ttl" : 50708,
      "_source" : {
        "application" : "test-framework-gru",
        "user_p" : 0.104166666666667,
        "hostname" : "10.31.33.168",
        "@timestamp" : "2019-11-01T06:38:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsVyDGb6oRN1s6Ret",
      "_score" : 1.0,
      "_ttl" : 54326,
      "_source" : {
        "application" : "data-poller",
        "user_p" : 3.85245901639348,
        "hostname" : "10.31.37.224",
        "@timestamp" : "2019-11-01T06:38:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsWoL_4fAvohBL9MO",
      "_score" : 1.0,
      "_ttl" : 57790,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.44067796610193,
        "hostname" : "10.31.34.128",
        "@timestamp" : "2019-11-01T06:37:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsEvBGb6oRN1s6RAS",
      "_score" : 1.0,
      "_ttl" : -15500,
      "_source" : {
        "application" : "ui-invoicing",
        "user_p" : 1.66666666666667,
        "hostname" : "10.31.32.14",
        "@timestamp" : "2019-11-01T06:37:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsEB1Gb6oRN1s6Q_e",
      "_score" : 1.0,
      "_ttl" : -18392,
      "_source" : {
        "application" : "io-engine",
        "user_p" : 26.0169491525422,
        "hostname" : "10.31.59.146",
        "@timestamp" : "2019-11-01T06:36:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsGeowEcMd9IvKqg_",
      "_score" : 1.0,
      "_ttl" : -8357,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 1.33333333333326,
        "hostname" : "10.31.37.16",
        "@timestamp" : "2019-11-01T06:37:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsTrPGb6oRN1s6RcY",
      "_score" : 1.0,
      "_ttl" : 45698,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 2.08333333333333,
        "hostname" : "10.31.33.83",
        "@timestamp" : "2019-11-01T06:38:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsjkqGb6oRN1s6R7E",
      "_score" : 1.0,
      "_ttl" : 110813,
      "_source" : {
        "application" : "test-framework-gru",
        "user_p" : 0.145833333333343,
        "hostname" : "10.31.33.168",
        "@timestamp" : "2019-11-01T06:39:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsjNIwEcMd9IvKrVp",
      "_score" : 1.0,
      "_ttl" : 109307,
      "_source" : {
        "application" : "test-framework-gru",
        "user_p" : 0.125000000000028,
        "hostname" : "10.31.35.196",
        "@timestamp" : "2019-11-01T06:39:00+00:00"
      }
    }, {
      "_index" : "dev-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4lsgkjGb6oRN1s6R3b",
      "_score" : 1.0,
      "_ttl" : 98518,
      "_source" : {
        "application" : "processing-engine",
        "user_p" : 15.0,
        "hostname" : "10.31.32.113",
        "@timestamp" : "2019-11-01T06:38:00+00:00"
      }
    } ]
  }
}


curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"processing-engine","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-11-01T07:45:00+00:00"}'


curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "data-poller","user_p" : 3.85245901639348,"hostname" : "10.31.37.224","@timestamp" : "2019-11-01T06:38:00+00:00"}'


curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "ui-heimdall","user_p" : 2.08333333333333,"hostname" : "10.31.33.83","@timestamp" : "2019-11-01T06:38:00+00:00"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application":"processing-engine","user_p":0.666666666666664,"hostname":"10.31.33.232","@timestamp":"2019-11-06T07:21:07+00:00"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "data-poller","user_p" : 3.85245901639348,"hostname" : "10.31.37.224","@timestamp" : "2019-11-06T07:08:24+00:00"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "data-poller","user_p" : 3.85245901639348,"hostname" : "10.31.37.224","@timestamp" : "2019-11-06T07:10:24+00:00"}'


curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "data-poller","user_p" : 3.85245901639348,"hostname" : "10.31.37.224","@timestamp" : "2019-11-06T07:12:24+00:00"}'

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "io-engine","user_p" : 3.9245901639348,"hostname" : "10.31.37.52","@timestamp" : "2019-11-06T07:12:24+00:00"}'
"application" : "ui-heimdall",
        "user_p" : 2.08333333333333,
        "hostname" : "10.31.33.83",
        "@timestamp" : "2019-11-01T06:38:00+00:00"




1.10.2019
: Friday, November 1, 2019 1:15:00 PM GMT+05:30

{
    "query": {
        "match_all": {}
    },"size":10,
    "filter": {
        "term": {
          "status": "published"
        }
        },
    , "aggs":{ "output":{  "terms":{ "field":"user_p" }  } } 
}


curl -X GET "localhost:9200/_search?pretty" -H 'Content-Type: application/json' -d'
{
    "query": {
        "range" : {
            "timestamp" : {
                "gte": "2015-01-01 00:00:00", 
                "lte": "now", 
                "time_zone": "+01:00"
            }
        }
    }
}
'

 "filter": {
        "term": {
          "@timestamp": {
                "gte": "2019-10-29T00:00:00+", 
                "lte": "now", 
                "time_zone": "+05:30"
            }
        }
        }
    
{
    "data": [
        {
            "10.31.34.166-data-poller": 4.06779670715332,
            "10.31.35.156-test-framework-gru": 0.1458333283662796,
            "10.31.61.138-io-engine": 25,
            "10.31.36.86-jobs": 13.583333015441895,
            "10.31.36.242-ui-mex": 1.4166666269302368,
            "10.31.32.62-test-framework-gru": 0.125,
            "10.31.34.100-processing-engine": 17.711864471435547,
            "10.31.35.203-ui-heimdall": 1.75,
            "10.31.36.108-ui-invoicing": 17.91666603088379,
            "10.31.34.231-regapi-ws": 1.9166666269302368,
            "10.31.32.198-ui-heimdall": 1.25,
            "10.31.35.196-test-framework-gru": 0.1458333283662796,
            "10.31.36.48-ui-rc": 2.704918146133423,
            "10.31.36.87-ui-heimdall": 4.34426212310791,
            "10.31.34.191-ui-heimdall": 1.6666666269302368,
            "time": 1572860820000,
            "10.31.34.200-ui-heimdall": 2.9166667461395264
        },
        {
            "10.31.34.166-data-poller": 4.426229476928711,
            "10.31.35.156-test-framework-gru": 0.125,
            "10.31.61.138-io-engine": 25.491804122924805,
            "10.31.36.86-jobs": 14.5,
            "10.31.36.242-ui-mex": 1.6666666269302368,
            "10.31.32.62-test-framework-gru": 0.125,
            "10.31.35.203-ui-heimdall": 1.4166666269302368,
            "10.31.36.108-ui-invoicing": 4.166666507720947,
            "10.31.34.231-regapi-ws": 2.25,
            "10.31.32.198-ui-heimdall": 1.4166666269302368,
            "10.31.35.196-test-framework-gru": 0.125,
            "10.31.36.48-ui-rc": 15.93220329284668,
            "10.31.36.87-ui-heimdall": 2.8813560009002686,
            "10.31.34.191-ui-heimdall": 1.3934426307678223,
            "time": 1572860880000,
            "10.31.34.200-ui-heimdall": 1.5
        },
        {
            "10.31.34.166-data-poller": 4.166666507720947,
            "10.31.35.156-test-framework-gru": 0.16393442451953888,
            "10.31.61.138-io-engine": 25.67796516418457,
            "10.31.36.86-jobs": 13,
            "10.31.36.242-ui-mex": 2.0833332538604736,
            "10.31.32.62-test-framework-gru": 0.1666666716337204,
            "10.31.35.203-ui-heimdall": 1.5,
            "10.31.36.108-ui-invoicing": 1.9166666269302368,
            "10.31.34.231-regapi-ws": 1.5573770999908447,
            "10.31.32.198-ui-heimdall": 1.25,
            "10.31.35.196-test-framework-gru": 0.125,
            "10.31.36.48-ui-rc": 1.6393442153930664,
            "10.31.36.87-ui-heimdall": 3.8333332538604736,
            "10.31.34.191-ui-heimdall": 1.4406780004501343,
            "time": 1572860940000,
            "10.31.34.200-ui-heimdall": 1.5833333730697632
        },
        {
            "10.31.34.166-data-poller": 4.322033882141113,
            "10.31.35.156-test-framework-gru": 0.12711864709854126,
            "10.31.61.138-io-engine": 27.83333396911621,
            "10.31.36.86-jobs": 14.25,
            "10.31.36.242-ui-mex": 1.5573770999908447,
            "10.31.32.62-test-framework-gru": 0.1458333283662796,
            "10.31.35.203-ui-heimdall": 1.5,
            "10.31.36.108-ui-invoicing": 2.0833332538604736,
            "10.31.34.231-regapi-ws": 1.6949152946472168,
            "10.31.32.198-ui-heimdall": 1.4166666269302368,
            "10.31.35.196-test-framework-gru": 0.1458333283662796,
            "10.31.36.48-ui-rc": 1.9491525888442993,
            "10.31.36.87-ui-heimdall": 6.666666507720947,
            "10.31.34.191-ui-heimdall": 1.4166666269302368,
            "time": 1572861000000,
            "10.31.34.200-ui-heimdall": 1.3333333730697632
        },
        {
            "10.31.34.166-data-poller": 4.333333492279053,
            "10.31.35.156-test-framework-gru": 0.125,
            "10.31.61.138-io-engine": 24.83333396911621,
            "10.31.36.86-jobs": 12.916666984558105,
            "10.31.36.242-ui-mex": 1.9491525888442993,
            "10.31.32.62-test-framework-gru": 0.1666666716337204,
            "10.31.34.100-processing-engine": 13.852458953857422,
            "10.31.35.203-ui-heimdall": 1.4166666269302368,
            "10.31.36.108-ui-invoicing": 2.0833332538604736,
            "10.31.34.231-regapi-ws": 1.7213114500045776,
            "10.31.32.198-ui-heimdall": 1.6666666269302368,
            "10.31.35.196-test-framework-gru": 0.125,
            "10.31.36.48-ui-rc": 1.6666666269302368,
            "10.31.36.87-ui-heimdall": 10.25,
            "10.31.34.191-ui-heimdall": 1.2295081615447998,
            "time": 1572861060000,
            "10.31.34.200-ui-heimdall": 1.6666666269302368
        },
        {
            "10.31.34.166-data-poller": 4.836065769195557,
            "10.31.35.156-test-framework-gru": 0.1458333283662796,
            "10.31.61.138-io-engine": 24.83333396911621,
            "10.31.36.86-jobs": 15.5,
            "10.31.36.242-ui-mex": 1.4166666269302368,
            "10.31.32.62-test-framework-gru": 0.1458333283662796,
            "10.31.35.203-ui-heimdall": 2.3333332538604736,
            "10.31.36.108-ui-invoicing": 2.131147623062134,
            "10.31.34.231-regapi-ws": 1.7796610593795776,
            "10.31.32.198-ui-heimdall": 1.311475396156311,
            "10.31.35.196-test-framework-gru": 0.1458333283662796,
            "10.31.36.48-ui-rc": 2.25,
            "10.31.36.87-ui-heimdall": 4.75,
            "10.31.34.191-ui-heimdall": 1.6949152946472168,
            "time": 1572861120000,
            "10.31.34.200-ui-heimdall": 1.25
        },
        {
            "10.31.34.166-data-poller": 4.06779670715332,
            "10.31.35.156-test-framework-gru": 0.1458333283662796,
            "10.31.61.138-io-engine": 24.66666603088379,
            "10.31.36.86-jobs": 10.833333015441895,
            "10.31.36.242-ui-mex": 1.75,
            "10.31.32.62-test-framework-gru": 0.1458333283662796,
            "10.31.35.203-ui-heimdall": 1.9166666269302368,
            "10.31.36.108-ui-invoicing": 3.1355931758880615,
            "10.31.34.231-regapi-ws": 1.6666666269302368,
            "10.31.32.198-ui-heimdall": 1.4406780004501343,
            "10.31.35.196-test-framework-gru": 0.125,
            "10.31.36.48-ui-rc": 1.8333333730697632,
            "10.31.36.87-ui-heimdall": 5.416666507720947,
            "10.31.34.191-ui-heimdall": 1.4166666269302368,
            "time": 1572861180000,
            "10.31.34.200-ui-heimdall": 1.311475396156311
        },
        {
            "10.31.34.166-data-poller": 4.180327892303467,
            "10.31.35.156-test-framework-gru": 0.1458333283662796,
            "10.31.61.138-io-engine": 25.66666603088379,
            "10.31.36.86-jobs": 15.916666984558105,
            "10.31.36.242-ui-mex": 1.4166666269302368,
            "10.31.32.62-test-framework-gru": 0.125,
            "10.31.35.203-ui-heimdall": 1.5833333730697632,
            "10.31.36.108-ui-invoicing": 2.75,
            "10.31.34.231-regapi-ws": 1.4754098653793335,
            "10.31.32.198-ui-heimdall": 1.25,
            "10.31.35.196-test-framework-gru": 0.1458333283662796,
            "10.31.36.48-ui-rc": 1.8032786846160889,
            "10.31.36.87-ui-heimdall": 2.5409836769104004,
            "10.31.34.191-ui-heimdall": 1.3934426307678223,
            "time": 1572861240000,
            "10.31.34.200-ui-heimdall": 1.2711864709854126
        },
        {
            "10.31.34.166-data-poller": 3.8983051776885986,
            "10.31.35.156-test-framework-gru": 0.125,
            "10.31.61.138-io-engine": 26.16666603088379,
            "10.31.36.86-jobs": 15.083333015441895,
            "10.31.36.242-ui-mex": 1.6666666269302368,
            "10.31.32.62-test-framework-gru": 0.125,
            "10.31.34.100-processing-engine": 28.389829635620117,
            "10.31.35.203-ui-heimdall": 1.8333333730697632,
            "10.31.36.108-ui-invoicing": 3.4166667461395264,
            "10.31.34.231-regapi-ws": 1.8644068241119385,
            "10.31.32.198-ui-heimdall": 1.3333333730697632,
            "10.31.35.196-test-framework-gru": 0.125,
            "10.31.36.48-ui-rc": 1.7796610593795776,
            "10.31.36.87-ui-heimdall": 2.1186439990997314,
            "10.31.34.191-ui-heimdall": 1.4406780004501343,
            "time": 1572861300000,
            "10.31.34.200-ui-heimdall": 1.4166666269302368
        },
        {
            "10.31.34.166-data-poller": 4.016393661499023,
            "10.31.35.156-test-framework-gru": 0.1458333283662796,
            "10.31.61.138-io-engine": 25.33333396911621,
            "10.31.36.86-jobs": 13.916666984558105,
            "10.31.36.242-ui-mex": 1.9166666269302368,
            "10.31.32.62-test-framework-gru": 0.1666666716337204,
            "10.31.34.100-processing-engine": 15.166666984558105,
            "10.31.35.203-ui-heimdall": 1.5,
            "10.31.36.108-ui-invoicing": 2.5,
            "10.31.34.231-regapi-ws": 2.049180269241333,
            "10.31.32.198-ui-heimdall": 1.75,
            "10.31.35.196-test-framework-gru": 0.125,
            "10.31.36.48-ui-rc": 1.5,
            "10.31.36.87-ui-heimdall": 2.4166667461395264,
            "10.31.34.191-ui-heimdall": 1.311475396156311,
            "time": 1572861360000,
            "10.31.34.200-ui-heimdall": 1.5833333730697632
        },
        {
            "10.31.34.166-data-poller": 3.98305082321167,
            "10.31.35.156-test-framework-gru": 0.125,
            "10.31.61.138-io-engine": 26.065574645996094,
            "10.31.36.86-jobs": 13.833333015441895,
            "10.31.36.242-ui-mex": 2.6666667461395264,
            "10.31.32.62-test-framework-gru": 0.125,
            "10.31.35.203-ui-heimdall": 1.75,
            "10.31.36.108-ui-invoicing": 3,
            "10.31.34.231-regapi-ws": 2,
            "10.31.32.198-ui-heimdall": 1.3333333730697632,
            "10.31.35.196-test-framework-gru": 0.1458333283662796,
            "10.31.36.48-ui-rc": 1.5833333730697632,
            "10.31.36.87-ui-heimdall": 6.083333492279053,
            "10.31.34.191-ui-heimdall": 1.610169529914856,
            "time": 1572861420000,
            "10.31.34.200-ui-heimdall": 1.4166666269302368
        },
        {
            "10.31.34.166-data-poller": 4.34426212310791,
            "10.31.35.156-test-framework-gru": 0.125,
            "10.31.61.138-io-engine": 24.6610164642334,
            "10.31.36.86-jobs": 11.065573692321777,
            "10.31.36.242-ui-mex": 2,
            "10.31.32.62-test-framework-gru": 0.1458333283662796,
            "10.31.34.100-processing-engine": 14.166666984558105,
            "10.31.35.203-ui-heimdall": 1.4166666269302368,
            "10.31.36.108-ui-invoicing": 4.75,
            "10.31.34.231-regapi-ws": 2.203389883041382,
            "10.31.32.198-ui-heimdall": 1.3333333730697632,
            "10.31.35.196-test-framework-gru": 0.1458333283662796,
            "10.31.36.48-ui-rc": 1.5833333730697632,
            "10.31.36.87-ui-heimdall": 11.5,
            "10.31.34.191-ui-heimdall": 1.3333333730697632,
            "time": 1572861480000,
            "10.31.34.200-ui-heimdall": 1.25
        }
    ],
    "keys": [
        "10.31.61.138-io-engine",
        "10.31.36.108-ui-invoicing",
        "10.31.34.100-processing-engine",
        "10.31.36.86-jobs",
        "10.31.36.87-ui-heimdall",
        "10.31.34.166-data-poller",
        "10.31.34.200-ui-heimdall",
        "10.31.36.48-ui-rc",
        "10.31.34.231-regapi-ws",
        "10.31.35.203-ui-heimdall",
        "10.31.34.191-ui-heimdall",
        "10.31.36.242-ui-mex",
        "10.31.32.198-ui-heimdall",
        "10.31.35.156-test-framework-gru",
        "10.31.35.196-test-framework-gru",
        "10.31.32.62-test-framework-gru"
    ],
    "groupByItems": {
        "regapi-ws": "regapi-ws",
        "ui-mex": "ui-mex",
        "processing-engine": "processing-engine",
        "jobs": "jobs",
        "ui-invoicing": "ui-invoicing",
        "data-poller": "data-poller",
        "ui-rc": "ui-rc",
        "test-framework-gru": "test-framework-gru",
        "io-engine": "io-engine",
        "ui-heimdall": "ui-heimdall"
    }
}



{
    "data": {
        "10.31.33.83": "3.852458953857422",
        "10.31.37.224": "3.852458953857422",
        "10.31.33.232": "3.852458953857422"
    },
    "keys": {
        "10.31.33.83": "3.852458953857422",
        "10.31.37.224": "3.852458953857422",
        "10.31.33.232": "3.852458953857422"
    },
    "groupByItems": {
        "processing-engine": "processing-engine",
        "data-poller": "data-poller",
        "ui-heimdall": "ui-heimdall"
    }
}

using ESRestTemplate

from to logic for newrelic to get data from es
every 1 min taking old data also


{"data":{"10.31.33.83-processing-engine":2.0833332538604736,"10.31.37.224-ui-heimdall":3.852458953857422,"10.31.33.232-data-poller":0.6666666865348816},"keys":["10.31.33.232-data-poller","10.31.33.83-processing-engine","10.31.37.224-ui-heimdall"],"groupByItems":{"processing-engine":"processing-engine","data-poller":"data-poller","ui-heimdall":"ui-heimdall"}}

"application" : "io-engine",
        "user_p" : 26.0169491525422,
        "hostname" : "10.31.59.146",
        "@timestamp" : "2019-11-01T06:36:00+00:00"
      }

curl -X POST "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/system" -H 'Content-Type: application/json' -d'{"application" : "io-engine","user_p" : 26.0169491525422,"hostname" : "10.31.59.146","@timestamp" : "2019-11-01T06:36:00+00:00"}'


{"query":{
    "range":{
        "@timestamp" : {
            "gte": "2019-10-29T06:59:39+00:00", 
                "lte": "2019-10-30T13:17:43+00:00", 
                "time_zone": "+01:00"
        }
        }
    }
}


curl -X POST "localhost:9200/sales/_search?size=0&pretty" -H 'Content-Type: application/json' -d'
{
    "aggs" : {
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
"min":"2019-10-29T06:59:39+00:00",
"max":"2019-10-30T13:17:43+00:00"
            }
        }
    }
}
'

{
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
                "extended_bounds":{
        "min":"2019-10-29T06:59:39+00:00",
        "max":"2019-10-30T13:17:43+00:00"
                }
            }
        }
    }
}

{ \"match_all\":{} },\"size\":0, \"aggs\":{ \"output\":{  \"terms\":{ \"field\":\""+objectName+"\" }  } } 


{"query":{
    "match_all":{}},
    "size":0,
    "aggs":{
        "output":{
            "terms":{
                "field":"application"
                }
            }
        }
    }


local


{
  "took": 0,
  "timed_out": false,
  "_shards": {
    "total": 1,
    "successful": 1,
    "skipped": 0,
    "failed": 0
  },
  "hits": {
    "total": 4,
    "max_score": 1,
    "hits": [
      {
        "_index": "dev-newrelic-cpu-utilization",
        "_type": "system",
        "_id": "AW4cvSAMmevqRYxNzU0u",
        "_score": 1,
        "_source": {
          "id": "1",
          "application": "test-framework-minion",
          "percent": 0.403333333333338,
          "user_p": 0.749999999999993,
          "hostname": "ip-10-31-59-78.awsdsi.ghx.com",
          "hostIp": "10.31.59.78",
          "@timestamp": "2019-10-29T06:59:39+00:00"
        }
      },
      {
        "_index": "dev-newrelic-cpu-utilization",
        "_type": "system",
        "_id": "AW4c0-WKmevqRYxNzU0w",
        "_score": 1,
        "_source": {
          "id": "1",
          "application": "test-framework-minion",
          "percent": 0.403333333333338,
          "user_p": 0.749999999999993,
          "hostname": "ip-10-31-59-78.awsdsi.ghx.com",
          "hostIp": "10.31.59.78",
          "@timestamp": "2019-10-29T06:59:39+00:00"
        }
      },
      {
        "_index": "dev-newrelic-cpu-utilization",
        "_type": "system",
        "_id": "AW4c1JCSmevqRYxNzU0x",
        "_score": 1,
        "_source": {
          "id": "1",
          "application": "test-framework-minion",
          "percent": 0.403333333333338,
          "user_p": 0.749999999999993,
          "hostname": "ip-10-31-59-78.awsdsi.ghx.com",
          "hostIp": "10.31.59.78",
          "@timestamp": "2019-10-30T13:17:43+00:00"
        }
      },
      {
        "_index": "dev-newrelic-cpu-utilization",
        "_type": "system",
        "_id": "AW4mjJ-o3vysBZWiw4FB",
        "_score": 1,
        "_source": {
          "application": "DEV-data-poller",
          "user_p": 0,
          "hostname": "10.31.33.240",
          "@timestamp": "2019-11-01T07:45:12.039Z"
        }
      }
    ]
  }
}


{"query":
{"bool":
{"must":{
"match_all":{}
},
"filter":{
"bool":{
"must":[{"range":
{"@timestamp":{
"gte":"2019-10-29T06:59:39+00:00",
"lte":"2019-10-30T13:17:43+00:00"
}
}
},{"term":{
"application":"All"
}
}
],"must_not":[{
"range":{"user_p":{
"lte":0}}}]
    
}}}},"size":0,
"aggs":{"items":{
"terms":{"field":"hostname"
,"size":100,
"order":{"count":"desc"}},
"aggs":{"count":{
"avg":{"field":"user_p"}}}}}}

{
  "query": {
    "bool": {
      "must": {
        "match_all": {}
      },
      "filter": {
        "bool": {
          "must": [
            {
              "range": {
                "@timestamp": {
                  "gte": "2019-10-29T06:59:39+00:00",
                  "lte": "2019-10-30T13:17:43+00:00"
                }
              }
            },
            {
              "term": {
                "application": "All"
              }
            }
          ],
          "must_not": [
            {
              "range": {
                "user_p": {
                  "lte": 0
                }
              }
            }
          ]
        }
      }
    }
  },
  "size": 0,
  "aggs": {
    "items": {
      "terms": {
        "field": "hostname",
        "size": 100,
        "order": {
          "count": "desc"
        }
      },
      "aggs": {
        "count": {
          "avg": {
            "field": "user_p"
          }
        }
      }
    }
  }
}


{\"bool\":{\"must\":{\"match_all\":{}},\"filter\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"gte\":"
                                + qFromDate + ",\"lte\":" + qToDate + "}}}


,{\"term\":{\"application\":\"" + filterType + "\"}}

"],\"must_not\":[{ \"range\": { \"user_p\": { \"lte\": 0 } } }]}}}}

"\"size\":0,\"aggs\":{\"output\":{\"date_histogram\":{\"field\":\"@timestamp\",\"interval\":"
                            + qInterval
                            + ",\"time_zone\":\"+05:30\",\"min_doc_count\":1,\"extended_bounds\":{\"min\":"
                            + qFromDate
                            + ",\"max\":"
                            + qToDate
                            + "}},\"aggs\":{\"items\":{\"terms\":{\"field\":\"hostname\",\"size\":100,\"order\":{\"count\":\"desc\"}},\"aggs\":{\"count\":{\"avg\":{\"field\":\"user_p\"}}}}}}}";



curl -X GET "localhost:9200/_search?pretty" -H 'Content-Type: application/json' -d'
{
    "query": {
        "match" : {
            "message" : {
                "query" : "to be or not to be",
                "operator" : "and",
                "zero_terms_query": "all"
            }
        }
    }
}
'
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'{
  "query": {
    "bool": {
      "must": {
        "match_all": {}
      },
      "filter": {
        "bool": {
          "must": [
            {
              "range": {
                "@timestamp": {
                  "gte": "2019-11-01T07:44:12.039Z",
                  "lte": "2019-11-01T07:56:07.429Z"
                }
              }
            },
            {
              "term": {
                "application": "ui-heimdall"
              }
            }
          ],
          "must_not": [
            {
              "range": {
                "user_p": {
                  "lte": 0
                }
              }
            }
          ]
        }
      }
    }
  },
  "size": 0,
  "aggs": {
    "items": {
      "terms": {
        "field": "hostname",
        "size": 100,
        "order": {
          "count": "desc"
        }
      },
      "aggs": {
        "count": {
          "avg": {
            "field": "user_p"
          }
        }
      }
    }
  }
}'

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'{
"query": {
"match_all":{}}
}'

curl -X POST "localhost:9200/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "bool" : {
      "must" : {
        "term" : { "application" : "All" }
      },
      "filter": {
        "term" : { "tag" : "tech" }
      },
      "must_not" : {
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:36:00+00:00", "lte" : "2019-11-01T07:45:00+00:00" }
        }
      },
      "should" : [
        { "term" : { "tag" : "wow" } },
        { "term" : { "tag" : "elasticsearch" } }
      ],
      "minimum_should_match" : 1,
      "boost" : 1.0
    }
  }
}
'
curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
    "bool" : {
      "must" : {
        "term" : { "application" : "ui-heimdall" }
      },
      "must" : {
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:36:00+00:00", "lte" : "2019-11-01T07:45:00+00:00" }
        }
      }
    }
  }
}
'

curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query": {
"bool": {
      "must" : {
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:36:00+00:00", "lte" : "2019-11-01T07:45:00+00:00" }
        }
      }
}
  }
}
'

curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query":{ "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"user_p" }  } }
}'

curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query":{ "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"hostname" }  } }
}'

curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{
  "query":{ "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } }
}'

curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
      "must" : {
          "terms":{ "application" : "ui-heimdall" },
        "range" : {
          "@timestamp" : { "gte" : "2019-10-29T06:59:39+00:00",
          "lte" : "2019-11-01T07:45:00+00:00" }
        }
      }
}
  },"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1
            }
        }
    }
}'


curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-10-29T06:59:39+00:00",
          "lte" : "2019-11-01T07:45:00+00:00" }
        }},
        {"term":{"application":"ui-heimdall"}}
      ]
}
  }
  }
  },"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1
            }
        }
    }
}'


curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-10-29T06:59:39+00:00",
          "lte" : "2019-11-01T07:45:00+00:00" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1
            }
        }
    }
}'


curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-10-29T06:59:39+00:00",
          "lte" : "2019-11-01T07:45:00+00:00" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-10-29T06:59:39+00:00",
"max":"2019-11-01T07:45:00+00:00"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
    }
}'


curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:36:00+00:00",
          "lte" : "2019-11-01T07:45:00+00:00" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-01T06:36:00+00:00",
"max":"2019-11-01T07:45:00+00:00"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
    }
}'
"2019-11-01T06:36:00+00:00", "lte" : "2019-11-01T07:45:00+00:00"

(base) sys-user@SMI-243:~/backup/elastic/cerebro-0.7.3/bin$ curl -X GET "search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty=true"
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 4,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW41OLyA_4fAvohBolss",
      "_score" : 1.0,
      "_ttl" : 70347962,
      "_source" : {
        "application" : "processing-engine",
        "user_p" : 0.666666666666664,
        "hostname" : "10.31.33.232",
        "@timestamp" : "2019-11-01T07:45:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW41OsWkwEcMd9IvnSp0",
      "_score" : 1.0,
      "_ttl" : 70481374,
      "_source" : {
        "application" : "data-poller",
        "user_p" : 3.85245901639348,
        "hostname" : "10.31.37.224",
        "@timestamp" : "2019-11-01T06:38:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW47n95GwEcMd9IvzsyX",
      "_score" : 1.0,
      "_ttl" : 177770112,
      "_source" : {
        "application" : "ui-heimdall",
        "user_p" : 2.08333333333333,
        "hostname" : "10.31.33.83",
        "@timestamp" : "2019-11-01T06:38:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW4_dq0rwEcMd9Iv72Ir",
      "_score" : 1.0,
      "_ttl" : 242179429,
      "_source" : {
        "application" : "io-engine",
        "user_p" : 26.0169491525422,
        "hostname" : "10.31.59.146",
        "@timestamp" : "2019-11-01T06:36:00+00:00"
      }
    } ]
  }
}


(base) sys-user@SMI-243:~/backup/elastic/cerebro-0.7.3/bin$ curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:36:00+00:00",
          "lte" : "2019-11-01T07:45:00+00:00" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-01T06:36:00+00:00",
"max":"2019-11-01T07:45:00+00:00"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
    }
}'
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "failed" : 0
  },
  "hits" : {
    "total" : 4,
    "max_score" : 0.0,
    "hits" : [ ]
  },
  "aggregations" : {
    "output" : {
      "buckets" : [ {
        "key_as_string" : "2019-11-01T12:06:00.000+05:30",
        "key" : 1572590160000,
        "doc_count" : 1,
        "items" : {
          "doc_count_error_upper_bound" : 0,
          "sum_other_doc_count" : 0,
          "buckets" : [ {
            "key" : "10.31.59.146",
            "doc_count" : 1,
            "count" : {
              "value" : 26.016948699951172
            }
          } ]
        }
      }, {
        "key_as_string" : "2019-11-01T12:08:00.000+05:30",
        "key" : 1572590280000,
        "doc_count" : 2,
        "items" : {
          "doc_count_error_upper_bound" : 0,
          "sum_other_doc_count" : 0,
          "buckets" : [ {
            "key" : "10.31.37.224",
            "doc_count" : 1,
            "count" : {
              "value" : 3.852458953857422
            }
          }, {
            "key" : "10.31.33.83",
            "doc_count" : 1,
            "count" : {
              "value" : 2.0833332538604736
            }
          } ]
        }
      }, {
        "key_as_string" : "2019-11-01T13:14:00.000+05:30",
        "key" : 1572594240000,
        "doc_count" : 1,
        "items" : {
          "doc_count_error_upper_bound" : 0,
          "sum_other_doc_count" : 0,
          "buckets" : [ {
            "key" : "10.31.33.232",
            "doc_count" : 1,
            "count" : {
              "value" : 0.6666666865348816
            }
          } ]
        }
      } ]
    }
  }
}

If filter type All for Cpu Utilization:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-11T08:15:00.404Z",
          "lte" : "2019-11-11T08:35:34.797Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-11T08:15:00.404Z",
"max":"2019-11-11T08:35:34.797Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-08T13:24:23.095Z",
          "lte" : "2019-11-08T13:35:07.652Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-08T13:24:23.095Z",
"max":"2019-11-08T13:35:07.652Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'


If filter type "application" for Cpu Utilization:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-11T08:15:00.404Z",
          "lte" : "2019-11-11T08:35:34.797Z" }
        }}
,{"term":{"application":"DEV-processing-engine"}}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-11T08:15:00.404Z",
"max":"2019-11-11T08:35:34.797Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'

https://internal-testfw-cxtf-1405-ui-heimdall-elb-2058419470.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/newrelic?chartGroup=osdao&chartName=CPU+Utilization+Process+Wise&filterType=All&fromDate=2019-11-11T12:24:22.404Z&interval=2m&testResultId=5dc95292d950101027b1d811&toDate=2019-11-11T12:36:18.797Z&viewType=chart

https://internal-testfw-cxtf-1405-ui-heimdall-elb-2058419470.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/?chartGroup=osdao&chartName=CPU+Utilization+Process+Wise&filterType=All&fromDate=2019-11-11T12:24:22.404Z&interval=2m&testResultId=5dc95292d950101027b1d811&toDate=2019-11-11T12:36:18.797Z&viewType=chart


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-11T12:24:22.404Z",
          "lte" : "2019-11-11T12:36:18.797Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
"terms":{"field":"beat.hostname","size":100},"aggs":{"inneroutput":{"date_histogram":{"field":"@timestamp","interval":"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-11T12:24:22.404Z","max":"2019-11-11T12:36:18.797Z"}},"aggs":{"items":{"terms":{"field":"proc.name","size":10,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"proc.cpu.user_p"}}}}}}}}}}'


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-11T12:24:22.404Z",
          "lte" : "2019-11-11T12:31:52.672Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
"terms":{"field":"beat.hostname","size":100},"aggs":{"inneroutput":{"date_histogram":{"field":"@timestamp","interval":"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-11T12:24:22.404Z","max":"2019-11-11T12:31:52.672Z"}},"aggs":{"items":{"terms":{"field":"proc.name","size":10,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"proc.cpu.user_p"}}}}}}}}}}'




"terms":{"field":"beat.hostname","size":100},"aggs":{"inneroutput":{"date_histogram":{"field":"@timestamp","interval":"2m","time_zone":"+05+30","min_doc_count":1,"extended_bounds":{"min":"","max":""}},"aggs":{"items":{"terms":{"field":"proc.name","size":0,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field:"proc.cpu.user_p"}}}}}}}}}




if (null != chartName && chartName.contains(CPU_UTILIZATION_PROCESS_WISE)) {
            aggregationsCurrVal =
                    "\"size\": 0,\"aggs\": {\"output\": {\"terms\": {\"field\": \"beat.hostname\",\"size\": 100},\"aggs\": {\"inneroutput\": {\"date_histogram\": {\"field\": \"@timestamp\",\"interval\" :"
                            + qInterval
                            + " ,\"time_zone\": \"+05:30\",\"min_doc_count\": 1,\"extended_bounds\": {\"min\": "
                            + qFromDate
                            + ",\"max\": "
                            + qToDate
                            + "}},\"aggs\": {\"items\": {\"terms\": {\"field\": \"proc.name\",\"size\": 10,\"order\": {\"count\": \"desc\"}},\"aggs\": {\"count\": {\"avg\": {\"field\": \"proc.cpu.user_p\"}}}}}}}}}";

https://corex-stg.awsdsi.ghx.com/rest/testframework/performance/metrics/?chartGroup=osdao&chartName=CPU+Utilization+Process+Wise&filterType=All&fromDate=2019-10-29T15:38:30.292Z&interval=2m&testResultId=5db85c933c4f1a1006cc7339&toDate=2019-10-29T15:56:52.881Z&viewType=chart
spalanichamy/Testingqwe123457#


After merging latest develop code , i got some db related error. you may get the same. 
you need add the below fields in AS2_profile table.

SSL_AUTH_ENABLED tinyint(1)
SSL_CERT_ID INT(11)
VERIFY_SERVER_CERT tinyint(1)

dev-testframework-stats_all
dev-testframework-stats

https://internal-testfw-cxtf-1405-ui-heimdall-elb-382767437.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/?chartGroup=jvmdao&chartName=Line+Chart+JVM+Heap+Stats&filterType=All&fromDate=2019-11-06T07:08:24.703Z&interval=2m&testResultId=5dc270f61c33fa102e215853&toDate=2019-11-06T07:21:07.227Z&viewType=chart

dev-testframework-stats-2019.10.18

{\"bool\":{\"must\":{\"match_all\":{}},\"filter\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"gte\":"
                                + qFromDate + ",\"lte\":" + qToDate + "}}}"

",{\"term\":{\"application\":\"" + filterType + "\"}}"

"],\"must_not\":[]}}}},\"size\":0,\"sort\": [{\"@timestamp\":{\"order\":\"desc\"}}]"

"\"aggs\":{\"output\":{\"terms\":{\"field\":\"host\",\"size\":100,\"order\":{\"count\":\"desc\"}},\"aggs\":{\"count\":{\"avg\":{\"field\":\"HeapMemory_max\"}},\"items\":{\"date_histogram\":{\"field\":\"@timestamp\",\"interval\":"
                            + qInterval
                            + ",\"min_doc_count\":1,\"extended_bounds\":{\"min\":"
                            + qFromDate
                            + ",\"max\":"
                            + qToDate
                            + "}},\"aggs\":{\"max\":{\"avg\":{\"field\":\"HeapMemory_max\"}},\"committed\":{\"avg\":{\"field\":\"HeapMemory_committed\"}},\"used\":{\"avg\":{\"field\":\"HeapMemory_used\"}}}}}}}";

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":
{"bool":{"must":{"match_all":{}},"filter":{
"bool":{
"must":[{"range":{"@timestamp":{"gte":"2019-11-04T06:47:56.633Z","lte":"2019-11-04T06:48:01.937Z"}}},
{"term":{"application":"processing-engine"}}],"must_not":[]}}}},"size":0,
"sort":[{"@timestamp":{"order":"desc"}}],
"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},"items":{"date_histogram":{"field":"@timestamp",
"interval":"2m","min_doc_count":1,"extended_bounds":{"min":"2019-11-04T06:47:56.633Z","max":"2019-11-04T06:48:01.937Z"}},
"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},
"used":{"avg":{"field":"HeapMemory_used"}}}}}}}
}'


(base) sys-user@SMI-243:~/backup/aws$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty=true"
{
  "took" : 4,
  "timed_out" : false,
  "_shards" : {
    "total" : 145,
    "successful" : 145,
    "failed" : 0
  },
  "hits" : {
    "total" : 729812,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "tpmdao-stats",
      "_id" : "AW41K6At_4fAvohBofoy",
      "_score" : 1.0,
      "_ttl" : 11908,
      "_source" : {
        "TPM_ClassName" : "DocTypeDAO",
        "@timestamp" : "2019-11-04T06:47:56.633Z",
        "instanceId" : "i-0efb354032e1a68b6",
        "application" : "processing-engine",
        "TPM_MethodName" : "findAll",
        "TPM_ClassName_MethodName" : "DocTypeDAO.findAll",
        "ghxhost" : "10.31.32.218",
        "stackname" : "CoreX-DEV-Processing-b6372",
        "TPM_ResponseTime" : 71
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "tpmdao-stats",
      "_id" : "AW41K6At_4fAvohBofo2",
      "_score" : 1.0,
      "_ttl" : 11908,
      "_source" : {
        "TPM_ClassName" : "ContactSubscriptionDAO",
        "@timestamp" : "2019-11-04T06:48:01.305Z",
        "instanceId" : "i-0efb354032e1a68b6",
        "application" : "processing-engine",
        "TPM_MethodName" : "getCronTZGroups",
        "TPM_ClassName_MethodName" : "ContactSubscriptionDAO.getCronTZGroups",
        "ghxhost" : "10.31.32.218",
        "stackname" : "CoreX-DEV-Processing-b6372",
        "TPM_ResponseTime" : 165
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "tpmdao-stats",
      "_id" : "AW41K6At_4fAvohBofo5",
      "_score" : 1.0,
      "_ttl" : 11908,
      "_source" : {
        "TPM_ClassName" : "ContactSubscriptionDAO",
        "@timestamp" : "2019-11-04T06:48:01.824Z",
        "instanceId" : "i-0efb354032e1a68b6",
        "application" : "processing-engine",
        "TPM_MethodName" : "getJobSubscriptions",
        "TPM_ClassName_MethodName" : "ContactSubscriptionDAO.getJobSubscriptions",
        "ghxhost" : "10.31.32.218",
        "stackname" : "CoreX-DEV-Processing-b6372",
        "TPM_ResponseTime" : 342
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "tpmdao-stats",
      "_id" : "AW41K6At_4fAvohBofo6",
      "_score" : 1.0,
      "_ttl" : 11908,
      "_source" : {
        "TPM_ClassName" : "ContactSubscriptionDAO",
        "@timestamp" : "2019-11-04T06:48:01.825Z",
        "instanceId" : "i-0efb354032e1a68b6",
        "application" : "processing-engine",
        "TPM_MethodName" : "getJobSubscriptions",
        "TPM_ClassName_MethodName" : "ContactSubscriptionDAO.getJobSubscriptions",
        "ghxhost" : "10.31.32.218",
        "stackname" : "CoreX-DEV-Processing-b6372",
        "TPM_ResponseTime" : 346
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "tpmdao-stats",
      "_id" : "AW41K6CWwEcMd9IvnLsX",
      "_score" : 1.0,
      "_ttl" : 12013,
      "_source" : {
        "TPM_ClassName" : "ContactSubscriptionDAO",
        "@timestamp" : "2019-11-04T06:48:01.937Z",
        "instanceId" : "i-097fb60d1a002e987",
        "application" : "jobs",
        "TPM_MethodName" : "getJobSubscriptions",
        "TPM_ClassName_MethodName" : "ContactSubscriptionDAO.getJobSubscriptions",
        "ghxhost" : "10.31.36.43",
        "stackname" : "CoreX-DEV-JOBS-b6372",
        "TPM_ResponseTime" : 223
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "mongo-stats",
      "_id" : "AW41K6EbGb6oRN1sWzJs",
      "_score" : 1.0,
      "_ttl" : 12146,
      "_source" : {
        "Mongo_ClassName_MethodName" : "EventBulkSupportImpl.executeBulk",
        "@timestamp" : "2019-11-04T06:47:02.648Z",
        "instanceId" : "i-0efb354032e1a68b6",
        "Mongo_MethodName" : "executeBulk",
        "Mongo_Aspect_Status" : "Error",
        "application" : "processing-engine",
        "Mongo_ResponseTimeMilliseconds" : 77,
        "Mongo_Aspect_Error_Reason" : "Invoked DAO is not an instance of Morphia's BasicDAO",
        "ghxhost" : "10.31.32.218",
        "stackname" : "CoreX-DEV-Processing-b6372",
        "Mongo_ClassName" : "EventBulkSupportImpl"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW41Kz_HGb6oRN1sWy7l",
      "_score" : 1.0,
      "_ttl" : -12770,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-04T06:47:36.542Z",
        "host" : "10.31.37.108",
        "type" : "jvm-heimdall-stats",
        "HeapMemory_committed" : 3758096384,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.37.108",
        "instanceId" : "i-001ee7d20d14c6863",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "2",
        "cfstack" : "TestFW-CoreX-CXTF-1405-UI-Heimdall-b2",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW41Kz_HGb6oRN1sWy7m",
      "_score" : 1.0,
      "_ttl" : -12770,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-04T06:47:36.542Z",
        "host" : "10.31.37.108",
        "type" : "jvm-heimdall-stats",
        "HeapMemory_max" : 3758096384,
        "service" : "corex-test-framework",
        "ghxhostcurl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : 2019-11-01T06:30:12.039Z,
          "lte" : 2019-11-01T07:56:07.429Z }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : 2m,
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":2019-11-01T07:44:12.039Z,
"max":2019-11-01T07:56:07.429Z
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'" : "10.31.37.108",
        "instanceId" : "i-001ee7d20d14c6863",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "2",
        "cfstack" : "TestFW-CoreX-CXTF-1405-UI-Heimdall-b2",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW41Kz_HGb6oRN1sWy7r",
      "_score" : 1.0,
      "_ttl" : -12770,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-04T06:47:36.553Z",
        "host" : "10.31.37.108",
        "type" : "jvm-heimdall-stats",
        "Threading_ThreadCount" : 148,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.37.108",
        "instanceId" : "i-001ee7d20d14c6863",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "2",
        "cfstack" : "TestFW-CoreX-CXTF-1405-UI-Heimdall-b2",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.04",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW41Kz_HGb6oRN1sWy7s",
      "_score" : 1.0,
      "_ttl" : -12770,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-04T06:47:36.554Z",
        "host" : "10.31.37.108",
        "type" : "jvm-heimdall-stats",
        "Threading_TotalStartedThreadCount" : 542,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.37.108",
        "instanceId" : "i-001ee7d20d14c6863",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "2",
        "cfstack" : "TestFW-CoreX-CXTF-1405-UI-Heimdall-b2",
        "application" : "ui-heimdall"
      }
    } ]
  }
}


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : 2019-11-01T06:30:12.039Z,
          "lte" : 2019-11-01T07:56:07.429Z }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : 2m,
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":2019-11-01T07:44:12.039Z,
"max":2019-11-01T07:56:07.429Z
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:30:12.039Z",
          "lte" : "2019-11-01T07:56:07.429Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-01T07:44:12.039Z",
"max":"2019-11-01T07:56:07.429Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'


new StringBuilder(
                        "{\"bool\":{\"must\":{\"match_all\":{}},\"filter\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"gte\":"
                                + qFromDate + ",\"lte\":" + qToDate + "}}}");

            commonQuery.append("],\"must_not\":[{ \"range\": { \"proc.cpu.user_p\": { \"lte\": 0 } } }]}}}}");


aggregationsCurrVal =
                    "\"size\": 0,\"aggs\": {\"output\": {\"terms\": {\"field\": \"beat.hostname\",\"size\": 100},\"aggs\": {\"inneroutput\": {\"date_histogram\": {\"field\": \"@timestamp\",\"interval\" :"
                            + qInterval
                            + " ,\"time_zone\": \"+05:30\",\"min_doc_count\": 1,\"extended_bounds\": {\"min\": "
                            + qFromDate
                            + ",\"max\": "
                            + qToDate
                            + "}},\"aggs\": {\"items\": {\"terms\": {\"field\": \"proc.name\",\"size\": 10,\"order\": {\"count\": \"desc\"}},\"aggs\": {\"count\": {\"avg\": {\"field\": \"proc.cpu.user_p\"}}} }}} }}}";


With FilterType All:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-04T11:14:46.228Z",
          "lte" : "2019-11-04T11:35:26.072Z" }
        }}],"must_not":[{"range":{"proc.cpu.user_p":{"lte":0}}}]}}}},
"size":0,"aggs":{"output":{"terms":{"field":"beat.hostname","size":100},
"aggs":{"inneroutput":{"date_histogram":{"field":"@timestamp","interval":"2m",
"time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-04T11:14:46.228Z",
"max":"2019-11-04T11:35:26.072Z"}},"aggs":{"items":{"terms":{"field":"proc.name",
"size":10,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"proc.cpu.user_p"}}}}}}}}}}'

Our qury{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"beat.hostname","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"beat.name"}}}}}

indexStr is dev-topbeat
2019-11-07 10:48:20,161 |INFO [http-bio-8443-exec-108] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is {"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"beat.hostname","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"beat.name"}}}}}

2019-11-07 10:48:20,160 |INFO [http-bio-8443-exec-108] com.ghx.web.filter.security.CRLFFilter - Url &#x3a; https://internal-testfw-cxtf-1405-ui-heimdall-elb-84960814.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/?chartgroup=osdao&chartname=cpu+utilization+process+wise&filtertype=all&fromdate=2019-11-04t11:14:46.228z&interval=2m&testresultid=5dc007efe37efa100ad35e03&todate=2019-11-04t11:35:26.072z&viewtype=chart
2019-11-07 10:48:20,160 |INFO [http-bio-8443-exec-108] com.ghx.web.filter.security.CRLFFilter - CRLF enabled true
2019-11-07 10:48:20,161 |INFO [http-bio-8443-exec-108] c.g.h.service.GHXHeimdallSecurityAspect - user tfwapi@ghx.com (id=3206631) attempt to access com.ghx.testframework.api.rest.PerformanceMetricsRESTService.getPerformanceMetricsStat&#x28;[CPU Utilization Process Wise, osdao, 2019-11-04T11:14:46.228Z, 2019-11-04T11:35:26.072Z, 2m, chart, All, 5dc007efe37efa100ad35e03, null, null, null, null]&#x29;
Inside getSearchResultsFromES indexAliasNamedev-topbeat_allindexName>>indexType>>nullquery>>{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"beat.hostname","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"beat.name"}}}}}


dev-topbeat_all

dev-topbeat

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:30:12.039Z",
          "lte" : "2019-11-01T07:56:07.429Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-01T07:44:12.039Z",
"max":"2019-11-01T07:56:07.429Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'


For CPU_UTILIZATION_PROCESS_WISE With FilterType All:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-04T11:14:46.228Z",
          "lte" : "2019-11-04T11:35:26.072Z" }
        }}],"must_not":[{"range":{"proc.cpu.user_p":{"lte":0}}}]}}}},
"size":0,"aggs":{"output":{"terms":{"field":"beat.hostname","size":100},
"aggs":{"inneroutput":{"date_histogram":{"field":"@timestamp","interval":"2m",
"time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-04T11:14:46.228Z",
"max":"2019-11-04T11:35:26.072Z"}},"aggs":{"items":{"terms":{"field":"proc.name",
"size":10,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"proc.cpu.user_p"}}}}}}}}}}'

devcurl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-04T11:14:46.228Z",
          "lte" : "2019-11-04T11:35:26.072Z" }
        }}
,{"term":{"application":"ui-heimdall"}}
],"must_not":[{"range":{"proc.cpu.user_p":{"lte":0}}}]}}}},
"size":0,"aggs":{"output":{"terms":{"field":"beat.hostname","size":100},
"aggs":{"inneroutput":{"date_histogram":{"field":"@timestamp","interval":"2m",
"time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-04T11:14:46.228Z",
"max":"2019-11-04T11:35:26.072Z"}},"aggs":{"items":{"terms":{"field":"proc.name",
"size":10,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"proc.cpu.user_p"}}}}}}}}}}'

",{\"term\":{\"application\":\"" + filterType + "\"}}"

For CPU_UTILIZATION_PROCESS_WISE With FilterType "application":
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-topbeat_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}},{"term":{"beat.name":"ui-heimdall"}}],"must_not":[{ "range": { "proc.cpu.user_p": { "lte": 0 } } }]}}}},"size": 0,"aggs": {"output": {"terms": {"field": "beat.hostname","size": 100},"aggs": {"inneroutput": {"date_histogram": {"field": "@timestamp","interval" :"1m" ,"time_zone": "+05:30","min_doc_count": 1,"extended_bounds": {"min": "2019-11-04T11:14:46.228Z","max": "2019-11-04T11:35:26.072Z"}},"aggs": {"items": {"terms": {"field": "proc.name","size": 10,"order": {"count": "desc"}},"aggs": {"count": {"avg": {"field": "proc.cpu.user_p"}}}}}}}}}}'


Listener CACHE:Local :
2019-11-07 11:24:47,355 |INFO [http-bio-8443-exec-116] com.ghx.web.filter.security.CRLFFilter - Url &#x3a; https://internal-testfw-cxtf-1405-ui-heimdall-elb-84960814.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/?chartgroup=osdao&chartname=cpu+utilization+process+wise&filtertype=ui-heimdall&fromdate=2019-11-04t11:14:46.228z&interval=2m&testresultid=5dc007efe37efa100ad35e03&todate=2019-11-04t11:35:26.072z&viewtype=chart
2019-11-07 11:24:47,355 |INFO [http-bio-8443-exec-116] com.ghx.web.filter.security.CRLFFilter - CRLF enabled true
2019-11-07 11:24:47,356 |INFO [http-bio-8443-exec-116] c.g.h.service.GHXHeimdallSecurityAspect - user spkannan (id=3186733) attempt to access com.ghx.testframework.api.rest.PerformanceMetricsRESTService.getPerformanceMetricsStat&#x28;[CPU Utilization Process Wise, osdao, 2019-11-04T11:14:46.228Z, 2019-11-04T11:35:26.072Z, 2m, chart, ui-heimdall, 5dc007efe37efa100ad35e03, null, null, null, null]&#x29;
Inside getSearchResultsFromES indexAliasNamedev-topbeat_allindexName>>indexType>>nullquery>>{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"beat.hostname","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"beat.name"}}}}}
2019-11-07 11:24:47,356 |INFO [http-bio-8443-exec-116] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-topbeat
2019-11-07 11:24:47,356 |INFO [http-bio-8443-exec-116] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is {"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"beat.hostname","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"beat.name"}}}}}
>>>>>>>>>>>Our qury{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"beat.hostname","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"beat.name"}}}}}
2019-11-07 11:24:47,375 |INFO [http-bio-8443-exec-116] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - searchResult.getResponseMetaData&#x28;&#x29;&#x3a; {TotalShards=145, SuccessShards=145, TotalRecords=20294, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=9}
searchResult.getResponseMetaData()>>>>{TotalShards=145, SuccessShards=145, TotalRecords=20294, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=9}
>>postQueryToEScom.ghx.common.search.model.SearchResult@738d5f5c
>>>Inside getSearchResultsFromES searchResultcom.ghx.common.search.model.SearchResult@738d5f5c
>>>Inside getSearchResultsFromES searchResult.getAggregations(){output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ip-10-31-34-200.awsdsi.ghx.com, doc_count=1540, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1540}]}}, {key=ip-10-31-34-165.awsdsi.ghx.com, doc_count=1535, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1535}]}}, {key=ip-10-31-37-188.awsdsi.ghx.com, doc_count=1516, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-mex, doc_count=1516}]}}, {key=ip-10-31-35-25.awsdsi.ghx.com, doc_count=1496, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-invoicing, doc_count=1496}]}}, {key=ip-10-31-36-38.awsdsi.ghx.com, doc_count=1484, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=processing-engine, doc_count=1484}]}}, {key=ip-10-31-36-76.awsdsi.ghx.com, doc_count=1477, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-rc, doc_count=1477}]}}, {key=ip-10-31-36-254.awsdsi.ghx.com, doc_count=1466, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=regapi-ws, doc_count=1466}]}}, {key=ip-10-31-32-198.awsdsi.ghx.com, doc_count=1453, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1453}]}}, {key=ip-10-31-36-13.awsdsi.ghx.com, doc_count=1443, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1443}]}}, {key=ip-10-31-35-203.awsdsi.ghx.com, doc_count=1432, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1432}]}}, {key=ip-10-31-60-97.awsdsi.ghx.com, doc_count=1400, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=io-engine, doc_count=1400}]}}, {key=ip-10-31-35-169.awsdsi.ghx.com, doc_count=1365, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=data-poller, doc_count=1365}]}}, {key=ip-10-31-34-133.awsdsi.ghx.com, doc_count=1363, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=jobs, doc_count=1363}]}}, {key=ip-10-31-34-191.awsdsi.ghx.com, doc_count=1324, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1324}]}}]}}
results{output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ip-10-31-34-200.awsdsi.ghx.com, doc_count=1540, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1540}]}}, {key=ip-10-31-34-165.awsdsi.ghx.com, doc_count=1535, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1535}]}}, {key=ip-10-31-37-188.awsdsi.ghx.com, doc_count=1516, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-mex, doc_count=1516}]}}, {key=ip-10-31-35-25.awsdsi.ghx.com, doc_count=1496, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-invoicing, doc_count=1496}]}}, {key=ip-10-31-36-38.awsdsi.ghx.com, doc_count=1484, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=processing-engine, doc_count=1484}]}}, {key=ip-10-31-36-76.awsdsi.ghx.com, doc_count=1477, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-rc, doc_count=1477}]}}, {key=ip-10-31-36-254.awsdsi.ghx.com, doc_count=1466, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=regapi-ws, doc_count=1466}]}}, {key=ip-10-31-32-198.awsdsi.ghx.com, doc_count=1453, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1453}]}}, {key=ip-10-31-36-13.awsdsi.ghx.com, doc_count=1443, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1443}]}}, {key=ip-10-31-35-203.awsdsi.ghx.com, doc_count=1432, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1432}]}}, {key=ip-10-31-60-97.awsdsi.ghx.com, doc_count=1400, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=io-engine, doc_count=1400}]}}, {key=ip-10-31-35-169.awsdsi.ghx.com, doc_count=1365, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=data-poller, doc_count=1365}]}}, {key=ip-10-31-34-133.awsdsi.ghx.com, doc_count=1363, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=jobs, doc_count=1363}]}}, {key=ip-10-31-34-191.awsdsi.ghx.com, doc_count=1324, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1324}]}}]}}
resultList[{key=ip-10-31-34-200.awsdsi.ghx.com, doc_count=1540, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1540}]}}, {key=ip-10-31-34-165.awsdsi.ghx.com, doc_count=1535, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1535}]}}, {key=ip-10-31-37-188.awsdsi.ghx.com, doc_count=1516, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-mex, doc_count=1516}]}}, {key=ip-10-31-35-25.awsdsi.ghx.com, doc_count=1496, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-invoicing, doc_count=1496}]}}, {key=ip-10-31-36-38.awsdsi.ghx.com, doc_count=1484, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=processing-engine, doc_count=1484}]}}, {key=ip-10-31-36-76.awsdsi.ghx.com, doc_count=1477, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-rc, doc_count=1477}]}}, {key=ip-10-31-36-254.awsdsi.ghx.com, doc_count=1466, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=regapi-ws, doc_count=1466}]}}, {key=ip-10-31-32-198.awsdsi.ghx.com, doc_count=1453, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1453}]}}, {key=ip-10-31-36-13.awsdsi.ghx.com, doc_count=1443, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1443}]}}, {key=ip-10-31-35-203.awsdsi.ghx.com, doc_count=1432, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1432}]}}, {key=ip-10-31-60-97.awsdsi.ghx.com, doc_count=1400, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=io-engine, doc_count=1400}]}}, {key=ip-10-31-35-169.awsdsi.ghx.com, doc_count=1365, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=data-poller, doc_count=1365}]}}, {key=ip-10-31-34-133.awsdsi.ghx.com, doc_count=1363, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=jobs, doc_count=1363}]}}, {key=ip-10-31-34-191.awsdsi.ghx.com, doc_count=1324, inneroutput={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=ui-heimdall, doc_count=1324}]}}]
Inside getSearchResultsFromES indexAliasNamedev-topbeat_allindexName>>indexType>>processquery>>{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}},{"term":{"beat.name":"ui-heimdall"}}],"must_not":[{ "range": { "proc.cpu.user_p": { "lte": 0 } } }]}}}},"size": 0,"aggs": {"output": {"terms": {"field": "beat.hostname","size": 100},"aggs": {"inneroutput": {"date_histogram": {"field": "@timestamp","interval" :"1m" ,"time_zone": "+05:30","min_doc_count": 1,"extended_bounds": {"min": "2019-11-04T11:14:46.228Z","max": "2019-11-04T11:35:26.072Z"}},"aggs": {"items": {"terms": {"field": "proc.name","size": 10,"order": {"count": "desc"}},"aggs": {"count": {"avg": {"field": "proc.cpu.user_p"}}}}}}}}}
2019-11-07 11:24:47,375 |INFO [http-bio-8443-exec-116] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-topbeat/process
2019-11-07 11:24:47,376 |INFO [http-bio-8443-exec-116] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is {"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}},{"term":{"beat.name":"ui-heimdall"}}],"must_not":[{ "range": { "proc.cpu.user_p": { "lte": 0 } } }]}}}},"size": 0,"aggs": {"output": {"terms": {"field": "beat.hostname","size": 100},"aggs": {"inneroutput": {"date_histogram": {"field": "@timestamp","interval" :"1m" ,"time_zone": "+05:30","min_doc_count": 1,"extended_bounds": {"min": "2019-11-04T11:14:46.228Z","max": "2019-11-04T11:35:26.072Z"}},"aggs": {"items": {"terms": {"field": "proc.name","size": 10,"order": {"count": "desc"}},"aggs": {"count": {"avg": {"field": "proc.cpu.user_p"}}}}}}}}}
>>>>>>>>>>>Our qury{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-04T11:14:46.228Z","lte":"2019-11-04T11:35:26.072Z"}}},{"term":{"beat.name":"ui-heimdall"}}],"must_not":[{ "range": { "proc.cpu.user_p": { "lte": 0 } } }]}}}},"size": 0,"aggs": {"output": {"terms": {"field": "beat.hostname","size": 100},"aggs": {"inneroutput": {"date_histogram": {"field": "@timestamp","interval" :"1m" ,"time_zone": "+05:30","min_doc_count": 1,"extended_bounds": {"min": "2019-11-04T11:14:46.228Z","max": "2019-11-04T11:35:26.072Z"}},"aggs": {"items": {"terms": {"field": "proc.name","size": 10,"order": {"count": "desc"}},"aggs": {"count": {"avg": {"field": "proc.cpu.user_p"}}}}}}}}}


dev-newrelic-topbeat_all

dev-newrelic-topbeat

curl -X POST "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/_aliases?pretty" -H 'Content-Type: application/json' -d'
{
    "actions" : [
        { "add" : { "index" : "dev-newrelic-cpu-utilization", "alias" : "dev-newrelic-cpu-utilization_all" } }
    ]
}
'

Wednesday, November 6, 2019 12:50:00 PM GMT+05:30

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-01T06:30:12.039Z",
          "lte" : "2019-11-01T07:56:07.429Z" }
        }}
],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-01T07:44:12.039Z",
"max":"2019-11-01T07:56:07.429Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"user_p"}}}}}}
        }
}'


{"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-07T07:14:00.226Z","lte" : "2019-11-07T07:19:37.670Z"}}}],"must_not":[{ "range": { "user_p": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-07T07:14:00.226Z","max":"2019-11-07T07:19:37.670Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"user_p"}}}}}}}



(base) sys-user@SMI-243:~/backup/aws$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty"
{
  "took" : 4,
  "timed_out" : false,
  "_shards" : {
    "total" : 145,
    "successful" : 145,
    "failed" : 0
  },
  "hits" : {
    "total" : 691817,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW47oVGkwEcMd9IvztiP",
      "_score" : 1.0,
      "_ttl" : -21390,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-05T12:54:17.677Z",
        "host" : "10.31.32.198",
        "type" : "jvm-heimdall-stats",
        "Threading_TotalStartedThreadCount" : 28761,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.32.198",
        "instanceId" : "i-01779d0b2891eb907",
        "ghxenvironment" : "dev",
        "appversion" : "1.109.0",
        "buildnumber" : "4",
        "cfstack" : "TestFW-CoreX-CXT1373-UI-Heimdall-b4",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW47oVHuwEcMd9IvztiR",
      "_score" : 1.0,
      "_ttl" : -21316,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-05T12:54:17.747Z",
        "host" : "10.31.34.191",
        "type" : "jvm-heimdall-stats",
        "HeapMemory_committed" : 3758096384,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.34.191",
        "instanceId" : "i-047ec32ed46a05fdf",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "1",
        "cfstack" : "TestFW-CoreX-CX1373-UI-Heimdall-b1",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW47oVHuwEcMd9IvztiV",
      "_score" : 1.0,
      "_ttl" : -21316,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-05T12:54:17.749Z",
        "host" : "10.31.34.191",
        "type" : "jvm-heimdall-stats",
        "Class_TotalLoadedClassCount" : 174926,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.34.191",
        "instanceId" : "i-047ec32ed46a05fdf",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "1",
        "cfstack" : "TestFW-CoreX-CX1373-UI-Heimdall-b1",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW47oVHuwEcMd9IvztiX",
      "_score" : 1.0,
      "_ttl" : -21316,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-05T12:54:17.752Z",
        "host" : "10.31.34.191",
        "type" : "jvm-heimdall-stats",
        "Threading_ThreadCount" : 135,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.34.191",
        "instanceId" : "i-047ec32ed46a05fdf",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "1",
        "cfstack" : "TestFW-CoreX-CX1373-UI-Heimdall-b1",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "jvm-heimdall-stats",
      "_id" : "AW47oVNr_4fAvohB1Adp",
      "_score" : 1.0,
      "_ttl" : -20935,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-05T12:54:18.128Z",
        "host" : "10.31.34.126",
        "type" : "jvm-heimdall-stats",
        "HeapMemory_used" : 1817209168,
        "service" : "corex-test-framework",
        "ghxhost" : "10.31.34.126",
        "instanceId" : "i-0cc1e26bc15084b93",
        "ghxenvironment" : "dev",
        "appversion" : "1.109.15",
        "buildnumber" : "2",
        "cfstack" : "TestFW-CoreX-CX1445REL-UI-Heimdall-b2",
        "application" : "ui-heimdall"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "tpmdao-stats",
      "_id" : "AW47oTIQ_4fAvohB1Aci",
      "_score" : 1.0,
      "_ttl" : -29474,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-05T12:53:55.149Z",
        "instanceId" : "i-0afc79d7691e66b3f",
        "application" : "processing-engine",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.37.61",
        "stackname" : "CoreX-DEV-Processing-b6384",
        "TPM_ResponseTime" : 147
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "tpmdao-stats",
      "_id" : "AW47oTIQ_4fAvohB1Ack",
      "_score" : 1.0,
      "_ttl" : -29474,
      "_source" : {
        "TPM_ClassName" : "ContactDAO",
        "@timestamp" : "2019-11-05T12:54:02.113Z",
        "instanceId" : "i-0afc79d7691e66b3f",
        "application" : "processing-engine",
        "TPM_MethodName" : "findUnsubscribedFromChannel",
        "TPM_ClassName_MethodName" : "ContactDAO.findUnsubscribedFromChannel",
        "ghxhost" : "10.31.37.61",
        "stackname" : "CoreX-DEV-Processing-b6384",
        "TPM_ResponseTime" : 1079
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "tpmdao-stats",
      "_id" : "AW47oTIQ_4fAvohB1Acm",
      "_score" : 1.0,
      "_ttl" : -29474,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-05T12:54:05.155Z",
        "instanceId" : "i-0afc79d7691e66b3f",
        "application" : "processing-engine",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.37.61",
        "stackname" : "CoreX-DEV-Processing-b6384",
        "TPM_ResponseTime" : 153
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "tpmdao-stats",
      "_id" : "AW47oTImwEcMd9Ivzth_",
      "_score" : 1.0,
      "_ttl" : -29452,
      "_source" : {
        "TPM_ClassName" : "AlertPostponedMessageDAO",
        "@timestamp" : "2019-11-05T12:54:06.404Z",
        "instanceId" : "i-048347f58df0d8558",
        "application" : "jobs",
        "TPM_MethodName" : "isPostponedMessageExistsForAlert",
        "TPM_ClassName_MethodName" : "AlertPostponedMessageDAO.isPostponedMessageExistsForAlert",
        "ghxhost" : "10.31.34.252",
        "stackname" : "CoreX-DEV-JOBS-b6384",
        "TPM_ResponseTime" : 22
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.05",
      "_type" : "tpmdao-stats",
      "_id" : "AW47oTImwEcMd9IvztiE",
      "_score" : 1.0,
      "_ttl" : -29452,
      "_source" : {
        "TPM_ClassName" : "AlertPostponedMessageDAO",
        "@timestamp" : "2019-11-05T12:54:10.039Z",
        "instanceId" : "i-048347f58df0d8558",
        "application" : "jobs",
        "TPM_MethodName" : "isPostponedMessageExistsForAlert",
        "TPM_ClassName_MethodName" : "AlertPostponedMessageDAO.isPostponedMessageExistsForAlert",
        "ghxhost" : "10.31.34.252",
        "stackname" : "CoreX-DEV-JOBS-b6384",
        "TPM_ResponseTime" : 32
      }
    } ]
  }
}


Heap memory Line Chart
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-05T12:53:10.039Z","lte":"2019-11-05T12:54:18.039Z"}}}],
"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},
"items":{"date_histogram":{"field":"@timestamp","interval":"1","min_doc_count":1,"extended_bounds":{"min":
"2019-11-05T12:53:10.039Z","max":"2019-11-05T12:54:18.039Z"}},"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},"used":{"avg":{"field":"HeapMemory_used"}}}}}}}}'

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-07T14:51:36.719Z","lte":"2019-11-08T04:30:04.557Z"}}}],
"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},
"items":{"date_histogram":{"field":"@timestamp","interval":"1","min_doc_count":1,"extended_bounds":{"min":
"2019-11-07T14:51:36.719Z","max":"2019-11-08T04:30:04.557Z"}},"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},"used":{"avg":{"field":"HeapMemory_used"}}}}}}}}'


new StringBuilder(
                        "{\"bool\":{\"must\":{\"match_all\":{}},\"filter\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"gte\":"
                                + qFromDate + ",\"lte\":" + qToDate + "}}}");

        commonQuery.append("],\"must_not\":[]}}}},\"size\":0,\"sort\": [{\"@timestamp\":{\"order\":\"desc\"}}]");
 "\"aggs\":{\"output\":{\"terms\":{\"field\":\"host\",\"size\":100,\"order\":{\"count\":\"desc\"}},\"aggs\":{\"count\":{\"avg\":{\"field\":\"HeapMemory_max\"}},\"items\":{\"date_histogram\":{\"field\":\"@timestamp\",\"interval\":"
                            + qInterval
                            + ",\"min_doc_count\":1,\"extended_bounds\":{\"min\":"
                            + qFromDate
                            + ",\"max\":"
                            + qToDate
                            + "}},\"aggs\":{\"max\":{\"avg\":{\"field\":\"HeapMemory_max\"}},\"committed\":{\"avg\":{\"field\":\"HeapMemory_committed\"}},\"used\":{\"avg\":{\"field\":\"HeapMemory_used\"}}}}}}}";

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-05T12:53:10.039Z","lte":"2019-11-05T12:54:18.039Z"}}}],
"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},
"items":{"date_histogram":{"field":"@timestamp","interval":"1","min_doc_count":1,"extended_bounds":{"min":
"2019-11-05T12:53:10.039Z","max":"2019-11-05T12:54:18.039Z"}},"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},"used":{"avg":{"field":"HeapMemory_used"}}}}}}}}'

(base) sys-user@SMI-243:~/backup/aws$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty"
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 48,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GWvq9wEcMd9IvJsm9",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-fleet-DEV",
        "user_p" : 197,
        "hostname" : "10.31.36.208",
        "@timestamp" : "2019-11-07T14:48:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GWvq9wEcMd9IvJsnC",
      "_score" : 1.0,
      "_source" : {
        "application" : "DEV-processing-engine",
        "user_p" : 34.3,
        "hostname" : "10.31.35.166",
        "@timestamp" : "2019-11-07T14:50:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GXAz0Gb6oRN1s5N04",
      "_score" : 1.0,
      "_source" : {
        "application" : "DEV-data-poller",
        "user_p" : 4.5,
        "hostname" : "10.31.33.224",
        "@timestamp" : "2019-11-07T14:49:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GXkaY_4fAvohBK_kG",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-fleet-DEV",
        "user_p" : 197,
        "hostname" : "10.31.36.208",
        "@timestamp" : "2019-11-07T14:51:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GX1naGb6oRN1s5Pco",
      "_score" : 1.0,
      "_source" : {
        "application" : "DEV-io-engine",
        "user_p" : 33.4,
        "hostname" : "10.31.61.193",
        "@timestamp" : "2019-11-07T14:54:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5Km9B7_4fAvohBTe_5",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-api-DEV",
        "user_p" : 0.588,
        "hostname" : "10.31.36.197",
        "@timestamp" : "2019-11-08T10:38:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GWvq9wEcMd9IvJsm7",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-api-DEVINT",
        "user_p" : 0.633,
        "hostname" : "10.31.34.145",
        "@timestamp" : "2019-11-07T14:48:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GWvq9wEcMd9IvJsm8",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-api-DEVINT",
        "user_p" : 0.467,
        "hostname" : "10.32.33.18",
        "@timestamp" : "2019-11-07T14:48:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GXAz0Gb6oRN1s5N00",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-api-DEVINT",
        "user_p" : 0.633,
        "hostname" : "10.31.34.145",
        "@timestamp" : "2019-11-07T14:49:00+00:00"
      }
    }, {
      "_index" : "dev-newrelic-cpu-utilization",
      "_type" : "system",
      "_id" : "AW5GXAz0Gb6oRN1s5N01",
      "_score" : 1.0,
      "_source" : {
        "application" : "connectplus-api-DEVINT",
        "user_p" : 0.463,
        "hostname" : "10.32.33.18",
        "@timestamp" : "2019-11-07T14:49:00+00:00"
      }
    } ]
  }
}


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-05T12:53:10.039Z","lte":"2019-11-05T12:54:18.039Z"}}}],
"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},
"items":{"date_histogram":{"field":"@timestamp","interval":"1","min_doc_count":1,"extended_bounds":{"min":
"2019-11-05T12:53:10.039Z","max":"2019-11-05T12:54:18.039Z"}},"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},"used":{"avg":{"field":"HeapMemory_used"}}}}}}}}'


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-05T12:53:10.039Z","lte":"2019-11-05T12:54:18.039Z"}}}],
"must_not":[]}}}}}'


new StringBuilder(
                        "{\"bool\":{\"must\":{\"match_all\":{}},\"filter\":{\"bool\":{\"must\":[{\"range\":{\"@timestamp\":{\"gte\":"
                                + qFromDate + ",\"lte\":" + qToDate
                                + "}}}],\"must_not\":[]}}}},\"size\":0,\"sort\": [{\"@timestamp\":{\"order\":\"desc\"}}]");
        aggregationsCurrVal =
                "\"aggs\":{\"output\":{\"terms\":{\"field\":\"" + objectName
                        + "\",\"size\" : 1000},\"aggs\":{\"inneroutput\":{\"terms\":{\"field\":\""+targetName+"\"}}" + "}}}";
       
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-08T13:24:23.095Z","lte"
:"2019-11-08T13:35:07.652Z"}}}],"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{
"field":"hostname","size":1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}}'


For Jvm Heap memory Line Chart with filter type All:

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-08T13:24:23.095Z","lte":"2019-11-08T13:35:07.652Z"}}}],
"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},
"items":{"date_histogram":{"field":"@timestamp","interval":"2m","min_doc_count":1,"extended_bounds":{"min":
"2019-11-08T13:24:23.095Z","max":"2019-11-08T13:35:07.652Z"}},"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},"used":{"avg":{"field":"HeapMemory_used"}}}}}}}}'

For Jvm Heap memory Line Chart with filter type Application:

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty" -H 'Content-Type: application/json' -d'
{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-08T12:20:23.095Z","lte":"2019-11-08T13:35:07.652Z"}}},{"term":{"application":"ui-heimdall"}}],
"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"HeapMemory_max"}},
"items":{"date_histogram":{"field":"@timestamp","interval":"2m","min_doc_count":1,"extended_bounds":{"min":
"2019-11-08T13:24:23.095Z","max":"2019-11-08T13:35:07.652Z"}},"aggs":{"max":{"avg":{"field":"HeapMemory_max"}},"committed":{"avg":{"field":"HeapMemory_committed"}},"used":{"avg":{"field":"HeapMemory_used"}}}}}}}}'


(base) sys-user@SMI-243:~$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/_template/dev-cpu-utilization*?pretty"
{
  "dev-cpu-utilization_v1" : {
    "order" : 0,
    "template" : "dev-cpu-utilization*",
    "settings" : { },
    "mappings" : {
      "system" : {
        "_ttl" : {
          "default" : "3d",
          "enabled" : true
        },
        "properties" : {
          "hostname" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "application" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      }
    },
    "aliases" : {
      "dev-cpu-utilization_all" : { }
    }
  }
}

Creating Template:
curl -X PUT "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/_template/dev-newrelic-cpu-utilization*?pretty" -H 'Content-Type: application/json' -d'
{"template": "dev-newrelic-cpu-utilization*","settings": {"number_of_shards": "5","number_of_replicas": "1"},"mappings": {"system": {"_ttl": {"default": "3d","enabled": true},"properties": {"user_p": {"type": "float"},"hostname": {"analyzer": "keyword","type": "string"},"application": {"analyzer": "keyword","type": "string"}}}},"aliases": {"dev-newrelic-cpu-utilization_all": {}}}'


Getting template created:
(base) sys-user@SMI-243:~$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/_template/dev-newrelic-cpu-utilization*?pretty"
{
  "dev-newrelic-cpu-utilization*" : {
    "order" : 0,
    "template" : "dev-newrelic-cpu-utilization*",
    "settings" : {
      "index" : {
        "number_of_shards" : "5",
        "number_of_replicas" : "0"
      }
    },
    "mappings" : {
      "system" : {
        "_ttl" : {
          "default" : "3d",
          "enabled" : true
        },
        "properties" : {
          "hostname" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "application" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      }
    },
    "aliases" : {
      "dev-newrelic-cpu-utilization_all" : { }
    }
  }
}


creating Index
curl -X PUT "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'
{
    "settings" : {
        "index" : {
            "number_of_shards" : 5, 
            "number_of_replicas" : 1 
        }
    },
 "mappings" : {
      "system" : {
        "_ttl" : {
          "default" : "3d",
          "enabled" : true
        },
        "properties" : {
          "hostname" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "application" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      }
    },
}
'


curl -X PUT "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization?pretty" -H 'Content-Type: application/json' -d'
{
    "settings" : {
"index" : {
        "number_of_shards" : 5, 
            "number_of_replicas" : 1 
}
    },
     "mappings" : {
      "system" : {
        "_ttl" : {
          "default" : "3d",
          "enabled" : true
        },
        "properties" : {
          "hostname" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "application" : {
            "analyzer" : "keyword",
            "type" : "string"
          },
          "user_p" : {
            "type" : "float"
          }
        }
      }
    },
"aliases" : {
      "dev-newrelic-cpu-utilization_all" : { }
    }
}
'


curl -X PUT "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization?pretty" -H 'Content-Type: application/json' -d'
{"settings": {"index": {"number_of_shards": 5,"number_of_replicas": 1}},"mappings": {"system": {"_ttl": {"default": "3d","enabled": true},"properties": {"used_bytes_by_host": {"type": "float"},"used_mb_by_host": {"type": "float"},"total_used_mb": {"type": "float"},"hostname": {"analyzer": "keyword","type": "string"},"application": {"analyzer": "keyword","type": "string"}}}},"aliases": {"stg-newrelic-heap-utilization_all": {}}}'
2019-11-11T08:12:00+00:00
2019-11-11T08:18:00+00:00

For CPU Utilization application grouping:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-11T08:12:00.095Z","lte":"2019-11-11T08:18:00.652Z"}}}],"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{
"field":"hostname","size":1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}}'

(base) sys-user@SMI-243:~/backup/aws$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-testframework-stats_all/_search?pretty"
{
  "took" : 4,
  "timed_out" : false,
  "_shards" : {
    "total" : 160,
    "successful" : 160,
    "failed" : 0
  },
  "hits" : {
    "total" : 647899,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "jvm-ue-stats",
      "_id" : "AW5K-GSc_4fAvohBUKOl",
      "_score" : 1.0,
      "_ttl" : -15490,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-08T12:23:42.633Z",
        "host" : "10.31.34.62",
        "type" : "jvm-ue-stats",
        "HeapMemory_max" : 3758096384,
        "service" : "corex-dp",
        "ghxhost" : "10.31.34.62",
        "instanceId" : "i-0501072a7fb0e7fc6",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "6413",
        "cfstack" : "CoreX-DEV-DP-b6413",
        "application" : "data-poller"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "jvm-ue-stats",
      "_id" : "AW5K-GSc_4fAvohBUKOo",
      "_score" : 1.0,
      "_ttl" : -15490,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-08T12:23:42.636Z",
        "host" : "10.31.34.62",
        "type" : "jvm-ue-stats",
        "Class_TotalLoadedClassCount" : 183637,
        "service" : "corex-dp",
        "ghxhost" : "10.31.34.62",
        "instanceId" : "i-0501072a7fb0e7fc6",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "6413",
        "cfstack" : "CoreX-DEV-DP-b6413",
        "application" : "data-poller"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "jvm-ue-stats",
      "_id" : "AW5K-GSc_4fAvohBUKOs",
      "_score" : 1.0,
      "_ttl" : -15490,
      "_source" : {
        "@version" : "1",
        "@timestamp" : "2019-11-08T12:23:42.642Z",
        "host" : "10.31.34.62",
        "type" : "jvm-ue-stats",
        "Threading_PeakThreadCount" : 387,
        "service" : "corex-dp",
        "ghxhost" : "10.31.34.62",
        "instanceId" : "i-0501072a7fb0e7fc6",
        "ghxenvironment" : "dev",
        "appversion" : "1.110.0",
        "buildnumber" : "6413",
        "cfstack" : "CoreX-DEV-DP-b6413",
        "application" : "data-poller"
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-DCdGb6oRN1sCdeq",
      "_score" : 1.0,
      "_ttl" : -28801,
      "_source" : {
        "TPM_ClassName" : "AdapterDAO",
        "@timestamp" : "2019-11-08T12:23:12.768Z",
        "instanceId" : "i-09df1658adfb4fd1d",
        "application" : "processing-engine",
        "TPM_MethodName" : "findAllAdapters",
        "TPM_ClassName_MethodName" : "AdapterDAO.findAllAdapters",
        "ghxhost" : "10.31.34.64",
        "stackname" : "CoreX-DEV-Processing-b6413",
        "TPM_ResponseTime" : 5726
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-DCdGb6oRN1sCdet",
      "_score" : 1.0,
      "_ttl" : -28801,
      "_source" : {
        "TPM_ClassName" : "ContactSubscriptionDAO",
        "@timestamp" : "2019-11-08T12:23:20.106Z",
        "instanceId" : "i-09df1658adfb4fd1d",
        "application" : "processing-engine",
        "TPM_MethodName" : "getJobSubscriptions",
        "TPM_ClassName_MethodName" : "ContactSubscriptionDAO.getJobSubscriptions",
        "ghxhost" : "10.31.34.64",
        "stackname" : "CoreX-DEV-Processing-b6413",
        "TPM_ResponseTime" : 52
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-IBbwEcMd9IvS7-h",
      "_score" : 1.0,
      "_ttl" : -8387,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-08T12:23:34.663Z",
        "instanceId" : "i-09df1658adfb4fd1d",
        "application" : "processing-engine",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.34.64",
        "stackname" : "CoreX-DEV-Processing-b6413",
        "TPM_ResponseTime" : 133
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-IBbwEcMd9IvS7-m",
      "_score" : 1.0,
      "_ttl" : -8387,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-08T12:23:44.663Z",
        "instanceId" : "i-09df1658adfb4fd1d",
        "application" : "processing-engine",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.34.64",
        "stackname" : "CoreX-DEV-Processing-b6413",
        "TPM_ResponseTime" : 132
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-BQvwEcMd9IvS7tX",
      "_score" : 1.0,
      "_ttl" : -36079,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-08T12:22:42.410Z",
        "instanceId" : "i-0501072a7fb0e7fc6",
        "application" : "data-poller",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.34.62",
        "stackname" : "CoreX-DEV-DP-b6413",
        "TPM_ResponseTime" : 132
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-BQvwEcMd9IvS7te",
      "_score" : 1.0,
      "_ttl" : -36079,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-08T12:23:12.409Z",
        "instanceId" : "i-0501072a7fb0e7fc6",
        "application" : "data-poller",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.34.62",
        "stackname" : "CoreX-DEV-DP-b6413",
        "TPM_ResponseTime" : 130
      }
    }, {
      "_index" : "dev-testframework-stats-2019.11.08",
      "_type" : "tpmdao-stats",
      "_id" : "AW5K-AFkGb6oRN1sCdak",
      "_score" : 1.0,
      "_ttl" : -40890,
      "_source" : {
        "TPM_ClassName" : "RuleDAO",
        "@timestamp" : "2019-11-08T12:23:12.604Z",
        "instanceId" : "i-0b528dbda7775ce09",
        "application" : "jobs",
        "TPM_MethodName" : "getAllActiveRuleIDs",
        "TPM_ClassName_MethodName" : "RuleDAO.getAllActiveRuleIDs",
        "ghxhost" : "10.31.34.156",
        "stackname" : "CoreX-DEV-JOBS-b6413",
        "TPM_ResponseTime" : 133
      }
    } ]
  }
}


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-cpu-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{
"from":0,"size":100,
"query": {"match_all":{}}}}'

For stg-newrelic-heap-utilization with FilterType All:

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-12T05:10:00.095Z",
          "lte" : "2019-11-12T05:12:00.652Z" }
        }}
],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "1m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-12T05:10:00.095Z",
"max":"2019-11-12T05:12:00.652Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}
        }
}'

For stg-newrelic-heap-utilization with FilterType Application:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-12T14:15:00.095Z",
          "lte" : "2019-11-12T14:18:00.652Z" }
        }}
,{"term":{"application":"DEV-processing-engine"}}
],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "1m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-12T14:15:00.095Z",
"max":"2019-11-12T14:18:00.652Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}
        }
}'


curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-12T05:10:00.095Z",
          "lte" : "2019-11-12T05:12:00.652Z" }
        }},{"term":{"application":"STG-processing-engine"}}
],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "1m",
                "time_zone":"+05:30",
                "min_doc_count":0,
"extended_bounds":{
"min":"2019-11-12T05:10:00.095Z",
"max":"2019-11-12T05:12:00.652Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}
        }
}'
(base) sys-user@SMI-243:~$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization/_search?pretty"
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 43,
    "max_score" : 1.0,
    "hits" : [ {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eCSNjwEcMd9IvyY39",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:10:00+00:00",
        "hostname" : "10.31.36.92",
        "total_used_mb" : 1050,
        "application" : "connectplus-api-STG",
        "used_bytes_by_host" : 92100000,
        "used_mb_by_host" : 87.8
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eCmvs_4fAvohBzlVQ",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:10:00+00:00",
        "hostname" : "10.32.37.186",
        "total_used_mb" : 1980,
        "application" : "connectplus-api-STG",
        "used_bytes_by_host" : 173000000,
        "used_mb_by_host" : 165
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eCmvs_4fAvohBzlVX",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:10:00+00:00",
        "hostname" : "10.31.61.34",
        "total_used_mb" : 29600,
        "application" : "STG-io-engine",
        "used_bytes_by_host" : 2590000000,
        "used_mb_by_host" : 2470
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eCmvs_4fAvohBzlVe",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:11:00+00:00",
        "hostname" : "10.31.35.39",
        "total_used_mb" : 58100,
        "application" : "STG-processing-engine",
        "used_bytes_by_host" : 5080000000,
        "used_mb_by_host" : 4840
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eCmvs_4fAvohBzlVh",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:11:00+00:00",
        "hostname" : "10.31.32.60",
        "total_used_mb" : 55300,
        "application" : "STG-processing-engine",
        "used_bytes_by_host" : 4830000000,
        "used_mb_by_host" : 4600
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eC7SsGb6oRN1sh2DB",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:12:00+00:00",
        "hostname" : "10.31.34.148",
        "total_used_mb" : 24300,
        "application" : "STG-jobs",
        "used_bytes_by_host" : 2160000000,
        "used_mb_by_host" : 2060
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eC7SsGb6oRN1sh2C7",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:12:00+00:00",
        "hostname" : "10.31.34.97",
        "total_used_mb" : 23100,
        "application" : "STG-data-poller",
        "used_bytes_by_host" : 2020000000,
        "used_mb_by_host" : 1920
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eC7SsGb6oRN1sh2DC",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:12:00+00:00",
        "hostname" : "10.31.32.26",
        "total_used_mb" : 50000,
        "application" : "STG-processing-engine",
        "used_bytes_by_host" : 4370000000,
        "used_mb_by_host" : 4170
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eC7SsGb6oRN1sh2DE",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:12:00+00:00",
        "hostname" : "10.31.36.43",
        "total_used_mb" : 68400,
        "application" : "STG-processing-engine",
        "used_bytes_by_host" : 5980000000,
        "used_mb_by_host" : 5700
      }
    }, {
      "_index" : "stg-newrelic-heap-utilization",
      "_type" : "system",
      "_id" : "AW5eC7SsGb6oRN1sh2DF",
      "_score" : 1.0,
      "_source" : {
        "@timestamp" : "2019-11-12T05:12:00+00:00",
        "hostname" : "10.31.35.39",
        "total_used_mb" : 74100,
        "application" : "STG-processing-engine",
        "used_bytes_by_host" : 6480000000,
        "used_mb_by_host" : 6180
      }
    } ]
  }
}




curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
{"query": {
"bool": {
    "must":{"match_all":{}},
    "filter":{"bool":{
      "must" : [{
        "range" : {
          "@timestamp" : { "gte" : "2019-11-12T07:08:00.404Z",
          "lte" : "2019-11-12T07:11:00.797Z" }
        }}
,{"term":{"application":"connectplus-api-STG"}}
],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,
    "aggs" : {
        "output":{
            "date_histogram" : {
                "field" : "@timestamp",
                "interval" : "2m",
                "time_zone":"+05:30",
                "min_doc_count":1,
"extended_bounds":{
"min":"2019-11-12T07:08:00.404Z",
"max":"2019-11-12T07:11:00.797Z"
            }},"aggs":{
"items":{"terms":{"field":"hostname","size":100,
"order":{"count":"desc"}},
"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}
        }
}'


(base) sys-user@SMI-243:~$ curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'
> {"query": {
> "bool": {
>     "must":{"match_all":{}},
>     "filter":{"bool":{
>       "must" : [{
>         "range" : {
>           "@timestamp" : { "gte" : "2019-11-12T07:08:00.404Z",
>           "lte" : "2019-11-12T07:11:00.797Z" }
>         }}
> ,{"term":{"application":"connectplus-api-STG"}}
> ],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,
>     "aggs" : {
>         "output":{
>             "date_histogram" : {
>                 "field" : "@timestamp",
>                 "interval" : "2m",
>                 "time_zone":"+05:30",
>                 "min_doc_count":1,
> "extended_bounds":{
> "min":"2019-11-12T07:08:00.404Z",
> "max":"2019-11-12T07:11:00.797Z"
>             }},"aggs":{
> "items":{"terms":{"field":"hostname","size":100,
> "order":{"count":"desc"}},
> "aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}
>         }
> }'
{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 5,
    "successful" : 5,
    "failed" : 0
  },
  "hits" : {
    "total" : 4,
    "max_score" : 0.0,
    "hits" : [ ]
  },
  "aggregations" : {
    "output" : {
      "buckets" : [ {
        "key_as_string" : "2019-11-12T12:40:00.000+05:30",
        "key" : 1573542600000,
        "doc_count" : 4,
        "items" : {
          "doc_count_error_upper_bound" : 0,
          "sum_other_doc_count" : 0,
          "buckets" : [ {
            "key" : "10.32.37.186",
            "doc_count" : 2,
            "count" : {
              "value" : 2255.0
            }
          }, {
            "key" : "10.31.36.92",
            "doc_count" : 2,
            "count" : {
              "value" : 944.0
            }
          } ]
        }
      } ]
    }
  }
}
2019-11-12T07:08:00

https://qbox.io/blog/comprehensive-guide-to-buckets-aggregations-in-elasticsearch

 "gte" : "2019-11-12T07:08:00.404Z",
          "lte" : "2019-11-12T07:11:00.797Z"

For Heap Utilization application grouping:
curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/stg-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T07:08:00.404Z","lte":"2019-11-12T07:11:00.797Z"}}}],"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{
"field":"hostname","size":1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}}'

/*public static void main(String[] args) {
    Client client;
    try {
        client = TransportClient.builder().build().
                addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("localhost"), 9300));

        QueryBuilder qb = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("user_agent", ""));
        SearchResponse scrollResp = client.prepareSearch("demo_risk_data")
            .addSort(SortParseElement.DOC_FIELD_NAME, SortOrder.ASC)
            .setScroll(new TimeValue(60000))
            .setQuery(qb)
            .setSize(100).execute().actionGet();

        //Scroll until no hits are returned
        while (true) {
            //Break condition: No hits are returned
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }

            // otherwise read results
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                JSONObject value = new JSONObject(searchData.getSource());
                System.out.println(value.toString());
            }

            // prepare next query
            scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
        }
    }
}*/

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'{"query":{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T14:18:12.988Z","lte":"2019-11-12T14:20:07.811Z"}}}],"must_not":[]}}}},"size":0,"sort":[{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{
"field":"hostname","size":1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}}'

curl -X GET "https://search-test-framework-es2-rh4ypdodfobsgmd54nfsnphatm.us-west-2.es.amazonaws.com/dev-newrelic-heap-utilization/_search?pretty" -H 'Content-Type: application/json' -d'{"query":
{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T14:18:12.988Z","lte":"2019-11-12T14:20:07.811Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}}'

#########################2019-11-12 14:54:57,016 |INFO [http-bio-8443-exec-160] com.ghx.web.filter.security.CRLFFilter - Url &#x3a; https://internal-testfw-cxtf-1405-ui-heimdall-elb-1347314042.us-west-2.elb.amazonaws.com/rest/testframework/performance/metrics/newrelic?chartgroup=jvmdao&chartname=line+chart+jvm+heap+stats&filtertype=all&fromdate=2019-11-12t14:18:12.988z&interval=2m&testresultid=5dcabec2e309741040fa86da&todate=2019-11-12t14:20:07.811z&viewtype=chart
2019-11-12 14:54:57,016 |INFO [http-bio-8443-exec-160] com.ghx.web.filter.security.CRLFFilter - CRLF enabled true
2019-11-12 14:54:57,022 |INFO [http-bio-8443-exec-160] c.g.h.service.GHXHeimdallSecurityAspect - user tfwapi@ghx.com (id=3206631) attempt to access com.ghx.testframework.api.rest.PerformanceMetricsRESTService.getPerformanceMetricsStatForNewrelic&#x28;[Line Chart JVM Heap Stats, jvmdao, 2019-11-12T14:18:12.988Z, 2019-11-12T14:20:07.811Z, 2m, chart, All, 5dcabec2e309741040fa86da, null, null, null, null]&#x29;
>>Inside getObjectStackMappedForNewrelic fromDate2019-11-12T14:18:12.988ZtoDate2019-11-12T14:20:07.811ZosIndexAliasNamedev-newrelic-heap-utilization_allosIndexNamedev-newrelic-heap-utilizationindexTypenullobjectNamehosttargetNameapplication
query>>>{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T14:18:12.988Z","lte":"2019-11-12T14:20:07.811Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}
Inside getSearchResultsFromES indexAliasNamedev-newrelic-heap-utilization_allindexName>>indexType>>nullquery>>{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T14:18:12.988Z","lte":"2019-11-12T14:20:07.811Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}
2019-11-12 14:54:57,022 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-newrelic-heap-utilization
2019-11-12 14:54:57,022 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is {"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T14:18:12.988Z","lte":"2019-11-12T14:20:07.811Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}
>>>>>>>>>>>Our qury{"bool":{"must":{"match_all":{}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"2019-11-12T14:18:12.988Z","lte":"2019-11-12T14:20:07.811Z"}}}],"must_not":[]}}}},"size":0,"sort": [{"@timestamp":{"order":"desc"}}],"aggs":{"output":{"terms":{"field":"host","size" : 1000},"aggs":{"inneroutput":{"terms":{"field":"application"}}}}}
2019-11-12 14:54:57,040 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - searchResult.getResponseMetaData&#x28;&#x29;&#x3a; {TotalShards=5, SuccessShards=5, TotalRecords=12, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
searchResult.getResponseMetaData()>>>>{TotalShards=5, SuccessShards=5, TotalRecords=12, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
>>postQueryToEScom.ghx.common.search.model.SearchResult@27441d14
>>>Inside getSearchResultsFromES searchResultcom.ghx.common.search.model.SearchResult@27441d14
>>>Inside getSearchResultsFromES searchResult.getAggregations(){output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[]}}
results{output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[]}}
resultList[]
stackObjectMapList>>[]
????stackObjectMap{}
ipAddressStackMapped{}
chartNameLine Chart JVM Heap StatsfromDate2019-11-12T14:18:12.988ZtoDate2019-11-12T14:20:07.811Zinterval2mfilterType(application):All
query{"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
indexType>>>>>>>>>>>>>>>system
Inside getSearchResultsFromES indexAliasNamedev-newrelic-heap-utilization_allindexName>>indexType>>systemquery>>{"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
2019-11-12 14:54:57,040 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-newrelic-heap-utilization/system
2019-11-12 14:54:57,040 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is {"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
>>>>>>>>>>>Our qury{"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
2019-11-12 14:54:57,045 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - searchResult.getResponseMetaData&#x28;&#x29;&#x3a; {TotalShards=5, SuccessShards=5, TotalRecords=11, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
searchResult.getResponseMetaData()>>>>{TotalShards=5, SuccessShards=5, TotalRecords=11, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
>>postQueryToEScom.ghx.common.search.model.SearchResult@567f4571
>>>Inside getSearchResultsFromES searchResultcom.ghx.common.search.model.SearchResult@567f4571
>>>Inside getSearchResultsFromES searchResult.getAggregations(){output={buckets=[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]}}
results{output={buckets=[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]}}
resultList[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]
2019-11-12 14:54:57,045 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - viewType&#x3a; chart
2019-11-12 14:54:57,045 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - Before Transformation ES resultList size&#x3a; 2
resultList:[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]
chartName:Line Chart JVM Heap Stats
viewType:chart
objectStackMapped:{}
indexAliasName:dev-newrelic-heap-utilization_all
indexName:dev-newrelic-heap-utilization
objectName:application
map.get(KEY) in 3rd if{time=1573568280000}
buketsMap>>>>>{doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}
After processBucketList>>>>>
valuesList >>>>>[{10.31.37.188=28400.0, time=1573568280000}]
map.get(KEY) in 3rd if{time=1573568400000}
buketsMap>>>>>{doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}
After processBucketList>>>>>
valuesList >>>>>[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}]
transformedESMap >>>>>{data=[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}]}
transformedESMap after adding Data and key>>>>>{data=[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}], keys=[10.31.37.188, 10.31.32.204, 10.31.61.133, 10.31.35.120, 10.31.32.112, 10.31.34.157, 10.32.35.238, 10.32.33.18, 10.31.37.147, 10.31.34.92]}
Inside getSearchResultsFromES indexAliasNamedev-newrelic-heap-utilization_allindexName>>indexType>>nullquery>> { "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } } 
2019-11-12 14:54:57,045 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-newrelic-heap-utilization
2019-11-12 14:54:57,045 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is  { "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } } 
>>>>>>>>>>>Our qury { "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } } 
2019-11-12 14:54:57,049 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - searchResult.getResponseMetaData&#x28;&#x29;&#x3a; {TotalShards=5, SuccessShards=5, TotalRecords=285, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
searchResult.getResponseMetaData()>>>>{TotalShards=5, SuccessShards=5, TotalRecords=285, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
>>postQueryToEScom.ghx.common.search.model.SearchResult@17ef9a68
>>>Inside getSearchResultsFromES searchResultcom.ghx.common.search.model.SearchResult@17ef9a68
>>>Inside getSearchResultsFromES searchResult.getAggregations(){output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=connectplus-api-DEVINT, doc_count=66}, {key=connectplus-api-DEV, doc_count=41}, {key=DEV-data-poller, doc_count=34}, {key=DEV-processing-engine, doc_count=34}, {key=DEV-io-engine, doc_count=33}, {key=DEV-jobs, doc_count=33}, {key=connectplus-fleet-DEV, doc_count=22}, {key=connectplus-fleet-DEVINT, doc_count=22}]}}
results{output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=connectplus-api-DEVINT, doc_count=66}, {key=connectplus-api-DEV, doc_count=41}, {key=DEV-data-poller, doc_count=34}, {key=DEV-processing-engine, doc_count=34}, {key=DEV-io-engine, doc_count=33}, {key=DEV-jobs, doc_count=33}, {key=connectplus-fleet-DEV, doc_count=22}, {key=connectplus-fleet-DEVINT, doc_count=22}]}}
resultList[{key=connectplus-api-DEVINT, doc_count=66}, {key=connectplus-api-DEV, doc_count=41}, {key=DEV-data-poller, doc_count=34}, {key=DEV-processing-engine, doc_count=34}, {key=DEV-io-engine, doc_count=33}, {key=DEV-jobs, doc_count=33}, {key=connectplus-fleet-DEV, doc_count=22}, {key=connectplus-fleet-DEVINT, doc_count=22}]
2019-11-12 14:54:57,049 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - Transformed Elastic Search Map size&#x3a; 3
elasticSearchResultsTransformation:::::::::{data=[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}], keys=[10.31.37.188, 10.31.32.204, 10.31.61.133, 10.31.35.120, 10.31.32.112, 10.31.34.157, 10.32.35.238, 10.32.33.18, 10.31.37.147, 10.31.34.92], groupByItems={DEV-io-engine=DEV-io-engine, DEV-jobs=DEV-jobs, DEV-data-poller=DEV-data-poller, connectplus-fleet-DEVINT=connectplus-fleet-DEVINT, connectplus-api-DEV=connectplus-api-DEV, connectplus-fleet-DEV=connectplus-fleet-DEV, connectplus-api-DEVINT=connectplus-api-DEVINT, DEV-processing-engine=DEV-processing-engine}}
Inside getSearchResultsFromES indexAliasNamedev-newrelic-heap-utilization_allindexName>>indexType>>systemquery>>{"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
2019-11-12 14:54:57,049 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-newrelic-heap-utilization/system
2019-11-12 14:54:57,049 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is {"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
>>>>>>>>>>>Our qury{"bool": {"must":{"match_all":{}},"filter":{"bool":{ "must" : [{"range" : {"@timestamp" : { "gte" : "2019-11-12T14:18:12.988Z","lte" : "2019-11-12T14:20:07.811Z"}}}],"must_not":[{ "range": { "total_used_mb": { "lte": 0 } } }]}}}},"size":0,"aggs" : {"output":{"date_histogram" : {"field" : "@timestamp","interval" :"2m","time_zone":"+05:30","min_doc_count":1,"extended_bounds":{"min":"2019-11-12T14:18:12.988Z","max":"2019-11-12T14:20:07.811Z"}},"aggs":{"items":{"terms":{"field":"hostname","size":100,"order":{"count":"desc"}},"aggs":{"count":{"avg":{"field":"total_used_mb"}}}}}}}
2019-11-12 14:54:57,053 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - searchResult.getResponseMetaData&#x28;&#x29;&#x3a; {TotalShards=5, SuccessShards=5, TotalRecords=11, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
searchResult.getResponseMetaData()>>>>{TotalShards=5, SuccessShards=5, TotalRecords=11, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
>>postQueryToEScom.ghx.common.search.model.SearchResult@4b4be1b6
>>>Inside getSearchResultsFromES searchResultcom.ghx.common.search.model.SearchResult@4b4be1b6
>>>Inside getSearchResultsFromES searchResult.getAggregations(){output={buckets=[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]}}
results{output={buckets=[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]}}
resultList[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]
2019-11-12 14:54:57,053 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - viewType&#x3a; chart
2019-11-12 14:54:57,053 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - Before Transformation ES resultList size&#x3a; 2
resultList:[{key_as_string=2019-11-12T19:48:00.000+05:30, key=1573568280000, doc_count=1, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}}, {key_as_string=2019-11-12T19:50:00.000+05:30, key=1573568400000, doc_count=10, items={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}}]
chartName:Line Chart JVM Heap Stats
viewType:chart
objectStackMapped:{}
indexAliasName:dev-newrelic-heap-utilization_all
indexName:dev-newrelic-heap-utilization
objectName:application
map.get(KEY) in 3rd if{time=1573568280000}
buketsMap>>>>>{doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.37.188, doc_count=1, count={value=28400.0}}]}
After processBucketList>>>>>
valuesList >>>>>[{10.31.37.188=28400.0, time=1573568280000}]
map.get(KEY) in 3rd if{time=1573568400000}
buketsMap>>>>>{doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=10.31.32.204, doc_count=1, count={value=30200.0}}, {key=10.31.61.133, doc_count=1, count={value=28100.0}}, {key=10.31.37.188, doc_count=1, count={value=25300.0}}, {key=10.31.35.120, doc_count=1, count={value=22000.0}}, {key=10.31.32.112, doc_count=1, count={value=16300.0}}, {key=10.31.34.157, doc_count=1, count={value=6410.0}}, {key=10.32.35.238, doc_count=1, count={value=3630.0}}, {key=10.32.33.18, doc_count=1, count={value=3150.0}}, {key=10.31.37.147, doc_count=1, count={value=2070.0}}, {key=10.31.34.92, doc_count=1, count={value=1560.0}}]}
After processBucketList>>>>>
valuesList >>>>>[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}]
transformedESMap >>>>>{data=[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}]}
transformedESMap after adding Data and key>>>>>{data=[{10.31.37.188=28400.0, time=1573568280000}, {10.31.35.120=22000.0, 10.31.61.133=28100.0, 10.31.32.204=30200.0, 10.32.33.18=3150.0, 10.31.32.112=16300.0, 10.31.37.147=2070.0, 10.31.37.188=25300.0, time=1573568400000, 10.32.35.238=3630.0, 10.31.34.157=6410.0, 10.31.34.92=1560.0}], keys=[10.31.37.188, 10.31.32.204, 10.31.61.133, 10.31.35.120, 10.31.32.112, 10.31.34.157, 10.32.35.238, 10.32.33.18, 10.31.37.147, 10.31.34.92]}
Inside getSearchResultsFromES indexAliasNamedev-newrelic-heap-utilization_allindexName>>indexType>>nullquery>> { "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } } 
2019-11-12 14:54:57,053 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - indexStr is dev-newrelic-heap-utilization
2019-11-12 14:54:57,053 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - esquery is  { "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } } 
>>>>>>>>>>>Our qury { "match_all":{} },"size":0, "aggs":{ "output":{  "terms":{ "field":"application" }  } } 
2019-11-12 14:54:57,056 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - searchResult.getResponseMetaData&#x28;&#x29;&#x3a; {TotalShards=5, SuccessShards=5, TotalRecords=285, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
searchResult.getResponseMetaData()>>>>{TotalShards=5, SuccessShards=5, TotalRecords=285, ES_MAX_SCORE=0.0, FailedShards=0, TookTime=1}
>>postQueryToEScom.ghx.common.search.model.SearchResult@67b37163
>>>Inside getSearchResultsFromES searchResultcom.ghx.common.search.model.SearchResult@67b37163
>>>Inside getSearchResultsFromES searchResult.getAggregations(){output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=connectplus-api-DEVINT, doc_count=66}, {key=connectplus-api-DEV, doc_count=41}, {key=DEV-data-poller, doc_count=34}, {key=DEV-processing-engine, doc_count=34}, {key=DEV-io-engine, doc_count=33}, {key=DEV-jobs, doc_count=33}, {key=connectplus-fleet-DEV, doc_count=22}, {key=connectplus-fleet-DEVINT, doc_count=22}]}}
results{output={doc_count_error_upper_bound=0, sum_other_doc_count=0, buckets=[{key=connectplus-api-DEVINT, doc_count=66}, {key=connectplus-api-DEV, doc_count=41}, {key=DEV-data-poller, doc_count=34}, {key=DEV-processing-engine, doc_count=34}, {key=DEV-io-engine, doc_count=33}, {key=DEV-jobs, doc_count=33}, {key=connectplus-fleet-DEV, doc_count=22}, {key=connectplus-fleet-DEVINT, doc_count=22}]}}
resultList[{key=connectplus-api-DEVINT, doc_count=66}, {key=connectplus-api-DEV, doc_count=41}, {key=DEV-data-poller, doc_count=34}, {key=DEV-processing-engine, doc_count=34}, {key=DEV-io-engine, doc_count=33}, {key=DEV-jobs, doc_count=33}, {key=connectplus-fleet-DEV, doc_count=22}, {key=connectplus-fleet-DEVINT, doc_count=22}]
2019-11-12 14:54:57,057 |INFO [http-bio-8443-exec-160] c.g.t.a.s.impl.PerformanceMetricsServiceImpl - Transformed Elastic Search Map size&#x3a; 3
2019-11-12 14:54:57,057 |INFO [http-bio-8443-exec-160] c.g.h.service.GHXHeimdallSecurityAspect - Access to com.ghx.testframework.api.rest.PerformanceMetricsRESTService granted to getPerformanceMetricsStatForNewrelic.[Line Chart JVM Heap Stats, jvmdao, 2019-11-12T14:18:12.988Z, 2019-11-12T14:20:07.811Z, 2m, chart, All, 5dcabec2e309741040fa86da, null, null, null, null]&#x28;user tfwapi@ghx.com (id=3206631)&#x29;
2019-11-12 14:55:04,901 |INFO [http-bio-8443-exec-129] com.ghx.web.filter.security.CRLFFilter - Url &#x3a; 
2019-11-12 14:55:04,901 |INFO [http-bio-8443-exec-129] com.ghx.web.filter.security.CRLFFilter - CRLF enabled true
2019-11-12 14:55:04,902 |INFO [http-bio-8443-exec-129] com.ghx.tpm.GHXTpmSecurityAspect - user spkannan (id=3186733) attempt to access com.ghx.tpm.rest.system.monitoring.SystemStatusRESTService.stoppedAndThrottled&#x28;[]&#x29;
2019-11-12 14:55:04,903 |INFO [http-bio-8443-exec-129] com.ghx.tpm.GHXTpmSecurityAspect - user spkannan (id=3186733) attempt to access com.ghx.tpm.rest.system.monitoring.cache.ComponentAndAdapterStatusMonitoringConfigCacheImpl.getCacheNameKey&#x28;[]&#x29;
2019-11-12 14:55:04,903 |INFO [http-bio-8443-exec-129] com.ghx.tpm.GHXTpmSecurityAspect - Access to com.ghx.tpm.rest.system.monitoring.cache.ComponentAndAdapterStatusMonitoringConfigCacheImpl granted to getCacheNameKey.[]&#x28;user spkannan (id=3186733)&#x29;
2019-11-12 14:55:04,903 |INFO [http-bio-8443-exec-129] com.ghx.tpm.GHXTpmSecurityAspect - user spkannan (id=3186733) attempt to access com.ghx.tpm.rest.system.monitoring.cache.ComponentAndAdapterStatusMonitoringConfigCacheImpl.get&#x28;[componentAndAdapterStatusCache, com.ghx.tpm.rest.system.monitoring.SystemStatusRESTService$$Lambda$795/1382351673@3dfb6724]&#x29;
2019-11-12 14:55:04,907 |INFO [http-bio-8443-exec-129] com.ghx.tpm.GHXTpmSecurityAspect - user spkannan (id=3186733) attempt to access com.ghx.tpm.rest.system.monitoring.ComponentAdapterReportingServiceImpl.getCurrentComponentAndAdapterStatusReport&#x28;[com.ghx.system.monitoring.model.dto.StoppedListenersDto@1e341e62]&#x29;
2019-11-12 14:55:04,908 |INFO [http-bio-8443-exec-129] com.ghx.tpm.GHXTpmSecurityAspect - Acce



***********************************************************************************
jmeter:

package jmxRunTest;

import org.apache.jmeter.engine.StandardJMeterEngine;
import org.apache.jmeter.reporters.ResultCollector;
import org.apache.jmeter.reporters.Summariser;
import org.apache.jmeter.save.SaveService;
import org.apache.jmeter.util.JMeterUtils;
import org.apache.jorphan.collections.HashTree;

import java.io.File;
import java.io.FileInputStream;

public class jmeterRun {

public static void main(String[] argv) throws Exception {
// JMeter Engine
StandardJMeterEngine jmeter = new StandardJMeterEngine();

// Initialize Properties, logging, locale, etc.
JMeterUtils.loadJMeterProperties("/home/sys-user/test/bin/jmeter.properties");
JMeterUtils.setJMeterHome("/home/sys-user/test");
JMeterUtils.initLogging();// you can comment this line out to see extra log messages of i.e. DEBUG level
JMeterUtils.initLocale();

// Initialize JMeter SaveService
SaveService.loadProperties();

// Load existing .jmx Test Plan
HashTree testPlanTree = SaveService.loadTree(new File("/home/sys-user/checking_JMX/JMX/Karina_JMX/MDM_Bulk_Upload_v1.11.jmx"));
Summariser summer = null;
String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary");
if (summariserName.length() > 0) {
summer = new Summariser(summariserName);
}

// Store execution results into a .jtl file
String logFile = "/home/sys-user/Documents/test.jtl";
ResultCollector logger = new ResultCollector(summer);
logger.setFilename(logFile);
testPlanTree.add(testPlanTree.getArray()[0], logger);

// Run JMeter Test
jmeter.configure(testPlanTree);
jmeter.run();
}
}

 test.jtl:

timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,Latency,IdleTime,Connect
1544788684425,494,MDM_Bulk_1,Non HTTP response code: java.net.UnknownHostException,Non HTTP response message: corex-dev-regapi.awsdsi.ghx.com: Name or service not known,Thread Group 1-1,text,false,,2180,0,1,1,0,0,494
1544788684937,714,MDM_Response_Body_1,500,javax.script.ScriptException: java.net.UnknownHostException: internal-testfw-cxtf-1176-ui-heimdall-elb-1772446433.us-west-2.elb.amazonaws.com: Name or service not known,Thread Group 1-1,text,false,,0,0,1,1,0,0,0

***********************************************************************************


package mailsend;

import java.util.*;
import javax.mail.*;
import javax.mail.internet.*;

public class send {

private static String USER_NAME = "kspriyaece1995";  // GMail user name (just the part before "@gmail.com")
private static String PASSWORD = "haripriya"; // GMail password

private static String RECIPIENT = "selvapriya.k@mookambikainfo.com";

public static void main(String[] args) {
    String from = USER_NAME;
    String pass = PASSWORD;
    String[] to = { RECIPIENT }; // list of recipient email addresses
    String subject = "Java send mail example";
    String body = "hi ....,!";

    sendFromGMail(from, pass, to, subject, body);
}

private static void sendFromGMail(String from, String pass, String[] to, String subject, String body) {
    Properties props = System.getProperties();
  String host = "smtp.gmail.com";

    props.put("mail.smtp.starttls.enable", "true");

    props.put("mail.smtp.ssl.trust", host);
    props.put("mail.smtp.user", from);
    props.put("mail.smtp.password", pass);
    props.put("mail.smtp.port", "587");
    props.put("mail.smtp.auth", "true");


    Session session = Session.getDefaultInstance(props);
    MimeMessage message = new MimeMessage(session);

    try {


        message.setFrom(new InternetAddress(from));
        InternetAddress[] toAddress = new InternetAddress[to.length];

        // To get the array of addresses
        for( int i = 0; i < to.length; i++ ) {
            toAddress[i] = new InternetAddress(to[i]);
        }

        for( int i = 0; i < toAddress.length; i++) {
            message.addRecipient(Message.RecipientType.TO, toAddress[i]);
        }



        message.setSubject(subject);
        message.setText(body);


        Transport transport = session.getTransport("smtp");


        transport.connect(host, from, pass);
        transport.sendMessage(message, message.getAllRecipients());
        transport.close();

    }
    catch (AddressException ae) {
        ae.printStackTrace();
    }
    catch (MessagingException me) {
        me.printStackTrace();
    }
    }
   } 


***********************************************************************************


Java 8:

Short circuiting ,intermediate and terminal operation

https://www.javabrahman.com/java-8/understanding-java-8-streams-operations-intermediate-and-terminal-operations-tutorial-with-examples/

https://www.logicbig.com/tutorials/core-java-tutorial/java-util-stream/short-circuiting.html

optional – java 8

.orElseThrow

Example: Java String lastIndexOf(int ch) Method

Examples of Using Java’s CompletableFuture

forkjoinpool java

GWT - UiBinder


Weekly deletion:

import boto3
import os
import datetime
import calendar
from datetime import date
from dateutil.parser impor parse
ec2 = boto3.resource('ec2')

def lambda_handler(event, context):
    INSTANCE_TAG_FILTER_SERVICE = os.environ.get("INSTANCE_TAG_FILTER_SERVICE")
    INSTANCE_TAG_NAME_FILTER_FOR_MAX_DAYS_TO_HOLD = os.environ.get("INSTANCE_TAG_NAME_FILTER_FOR_MAX_DAYS_TO_HOLD")
    INSTANCE_TAG_NAME_FILTER_FOR_WEEK_DELETION = os.environ.get("INSTANCE_TAG_NAME_FILTER_FOR_WEEK_DELETION")
    MAX_DAYS_TO_HOLD_INSTANCE = os.environ.get("MAX_DAYS_TO_HOLD_INSTANCE")
    DELETION_PER_WEEK_ON = os.environ.get("DELETION_PER_WEEK_ON")
    FILTER_FOR_MAX_DAYS = INSTANCE_TAG_NAME_FILTER_FOR_MAX_DAYS_TO_HOLD.split(",")
    FILTER_FOR_WEEKLY_DELETION = INSTANCE_TAG_NAME_FILTER_FOR_WEEK_DELETION.split(",")
    print("Max_Holding_Days: "+str(MAX_DAYS_TO_HOLD_INSTANCE))
    print("For Max holding days ,Instance tag with Name Containing: "+str(INSTANCE_TAG_NAME_FILTER_FOR_MAX_DAYS_TO_HOLD))
    print("For Weekly deletion ,Instance tag with Name Containing: "+str(INSTANCE_TAG_NAME_FILTER_FOR_WEEK_DELETION)) 
    i = 0
    while i < len(FILTER_FOR_MAX_DAYS):
        FILTER_FOR_MAX_DAYS[i] = "*"+FILTER_FOR_MAX_DAYS[i]+"*"
        i += 1
    i = 0
    while i < len(FILTER_FOR_WEEKLY_DELETION):
        FILTER_FOR_WEEKLY_DELETION[i] = "*"+FILTER_FOR_WEEKLY_DELETION[i]+"*"
        i += 1
    instanceForWeeklyDeletion = ec2.instances.filter(Filters=[{'Name': 'tag:Name', 'Values': FILTER_FOR_WEEKLY_DELETION},
                                            {'Name': 'tag:service', 'Values': [INSTANCE_TAG_FILTER_SERVICE]}])
    instanceDeletion = []
    for instance in instanceForWeeklyDeletion:
            if calendar.day_name[datetime.datetime.today().weekday()] == DELETION_PER_WEEK_ON:
                print("Today is Saturday, Going to delete instances"+str(instance.id))
                instanceDeletion.append(instance.id)
    instanceForMaxDays = ec2.instances.filter(Filters=[{'Name': 'tag:Name', 'Values': FILTER_FOR_MAX_DAYS},
                                            {'Name': 'tag:service', 'Values': [INSTANCE_TAG_FILTER_SERVICE]}])
    for instance in instanceForMaxDays:
            a = (parse(date.isoformat(date.today()))) -(parse(date.isoformat(instance.launch_time)))
            if a.days > MAX_DAYS_TO_HOLD_INSTANCE:
                print("Going beyond max holding day:"+str(MAX_DAYS_TO_HOLD_INSTANCE)+", Going to delete instances: "+str(instance.id))
                instanceDeletion.append(instance.id)
    print('*************************************************************')
    print("The Instances that will be deleted is:"+str(instanceDeletion))
    print("Total: "+str(len(instanceDeletion)))
    if len(instanceDeletion) > 0:
        ec2.instances.filter(InstanceIds = instanceDeletion).terminate()
    print('*************************************************************')

          _______________________________________________________________________

'''
Created on 17-Jan-2019

@author: sys-user
'''
import urllib2
import os
#import boto3
import re
import traceback

def send_mail(SENDER, RECIPIENTS, SUBJECT, RESULT_MSG):
    CHARSET = 'UTF-8'
    #try:
      #  client = boto3.client('ses', 'us-east-1')
      #  response = client.send_email(Destination={'ToAddresses': RECIPIENTS},Message={'Body': {'Text': {'Charset': CHARSET, 'Data': RESULT_MSG}},'Subject': {'Charset': CHARSET, 'Data': SUBJECT}}, Source=SENDER)
   # except:
       # pass
    traceback.print_exc()

def lambda_handler(event, context):
    TEST_FRAMEWORK_ES_CLUSTER_URL           = os.environ.get("TEST_FRAMEWORK_ES_CLUSTER_URL")
    TEST_FRAMEWORK_ES_CLUSTER_ALLOWED_INDICES_COUNT    = int(os.environ.get("TEST_FRAMEWORK_ES_CLUSTER_ALLOWED_INDICES_COUNT"))
    TEST_FRAMEWORK_ES_CLUSTER_MAINTAINERS        = os.environ.get("TEST_FRAMEWORK_ES_CLUSTER_MAINTAINERS")
    getClusterIndicesRequest = urllib2.Request(TEST_FRAMEWORK_ES_CLUSTER_URL + "/_cat/indices/*?h=creation.date,creation.date.string,index")
    indicesDataConnection=urllib2.urlopen(getClusterIndicesRequest)
    rawIndicesData = indicesDataConnection.read().split("\n")
    pattern = re.compile("\s+")
    parsedIndicesData = filter(None,[(pattern.sub(',', indexData))[:-1] for indexData in rawIndicesData])
    indicesDataConnection.close()
    sortedParsedIndicesData=sorted(parsedIndicesData, key=lambda indexDataString: long(indexDataString.split(",")[0]))
    countOfIndices=len(sortedParsedIndicesData)
    toBeDeleted = countOfIndices-TEST_FRAMEWORK_ES_CLUSTER_ALLOWED_INDICES_COUNT
    deletedIndicesCounter=0
    deletedIndicesList=[]
    for indexDataString in sortedParsedIndicesData:
        if deletedIndicesCounter<toBeDeleted: 
            deleteClusterIndexRequest = urllib2.Request(TEST_FRAMEWORK_ES_CLUSTER_URL + "/" + indexDataString.split(",")[2])
            deleteClusterIndexRequest.get_method = lambda: 'DELETE'
            deleteClusterIndexRequestConnection = urllib2.urlopen(deleteClusterIndexRequest)
            deleteClusterIndexRequestConnection.close()
            deletedIndicesList.append(indexDataString)
        else:
            break
        deletedIndicesCounter=deletedIndicesCounter+1
        if len(deletedIndicesList) > 0 :
       # send_mail("els-alerts@ghx.com", TEST_FRAMEWORK_ES_CLUSTER_MAINTAINERS.split(","), "Test Framework ES Cluster Maintenance Update", "The following indices have been successfully deleted from the Test Framework ES Cluster : " + TEST_FRAMEWORK_ES_CLUSTER_URL + "\n\n" + "\n".join(deletedIndicesList))
    #else:
        #send_mail("els-alerts@ghx.com", TEST_FRAMEWORK_ES_CLUSTER_MAINTAINERS.split(","), "Test Framework ES Cluster Maintenance Update", "No index need to be deleted from the Test Framework ES Cluster : " + TEST_FRAMEWORK_ES_CLUSTER_URL + "\n\n")

___________________________________________________________________________________________________


import boto3
s3_client = boto3.client('s3')
   
   AmazonS3Client s3Client = new AmazonS3Client().withRegion(Regions.US_EAST_1);
__________________________________________________________________________________

'''
Created on 17-Jan-2019

@author: sys-user
'''
from pip._vendor.distlib._backport.tarfile import LENGTH_LINK
'''
Created on 19-Sep-2018

@author: sys-user
'''
import httplib 

import urllib2
import json 
import csv
SERVER = "127.0.0.1:9200"
maxCount=5
whitelist = "testframe6,testframe5";

results = []
results=[x for x in whitelist.split(',')]
print results
test= urllib2.Request("http://"+SERVER+"/_cat/indices/*?v&s=index&s=creation.date:desc")
test.add_header('Accept', 'application/json; charset=UTF-8')
data=urllib2.urlopen(test).read()
print data
output = json.loads(data)
print output
lengthOfIndices=len(output);
if lengthOfIndices>maxCount:
    print(lengthOfIndices-maxCount)
length = lengthOfIndices-maxCount
minCount =0
for x in output:
   # print length
    #print minCount
    if minCount<length and not(x['index'] in results):
                     #print row[0]
                    # print row[1]
                     #print row
         print minCount
         print length
         print(not(x['index'] in results))
         print( x['index'])
         print("deleting ")
         conn = httplib.HTTPConnection(SERVER)
         conn.request('DELETE', '/'+x['index']) 
         resp = conn.getresponse()
         content = resp.read()
         print content
    elif minCount<length and x['index'] in results:
            minCount=minCount-1
    else:
        break
    minCount=minCount+1

*****************************************


freemarker find and replace:

public class FindAndReplaceTwo implements TestFrameworkAction ,TemplateExceptionHandler{
private static final String CUSTOM_ACTION = "Custom String";
public static void main(String args[]) {
try {
List<FindAndReplaceEntry> findAndReplace = new ArrayList<FindAndReplaceEntry>();

DocumentContext docContext = new DocumentContext();
        docContext.addItem(DocumentContext.Key.DOCUMENT_INPUT_FILENAME, "/home/sys-user/Desktop/testDocument.edi"); 
docContext.addItem(DocumentContext.Key.OUTBOUND_FILE_NAME, "/home/sys-user/Desktop/testDocument.edi");

if (docContext != null && docContext.contains(DocumentContext.Key.DOCUMENT_INPUT_FILENAME)) {

String inputFileName = docContext.getItem(DocumentContext.Key.DOCUMENT_INPUT_FILENAME);
LOGGER.info("Started performing find and replace action for input file name {}", inputFileName);
File file = new File(inputFileName);

Configuration configuration = new Configuration(Configuration.VERSION_2_3_20);
configuration.setLocalizedLookup(false);
configuration .setTemplateExceptionHandler(TemplateExceptionHandler.IGNORE_HANDLER);
FileTemplateLoader fileTemplateLoader = new FileTemplateLoader(
new File(file.getParent() + File.separator));
configuration.setTemplateLoader(fileTemplateLoader);
configuration.setObjectWrapper(new DefaultObjectWrapper());
Template template = configuration.getTemplate(file.getName());
FindAndReplaceEntry f1 = new FindAndReplaceEntry("poNumber", "123", null);
FindAndReplaceEntry f2 = new FindAndReplaceEntry("MOID", "abc", null);
findAndReplace.add(f1);
findAndReplace.add(f2);
Map<String, Object> replaceMapEntries = new HashMap<>();

                  for(int i=0;i<findAndReplace.size();) {
FindAndReplaceEntry findAndReplaceEntry =findAndReplace.get(i) ;

LOGGER.info("Find and replace entry...{}", findAndReplaceEntry);
if (findAndReplaceEntry != null) {
String action = CUSTOM_ACTION;
if (action != null && action.equalsIgnoreCase(CUSTOM_ACTION)) {
         if (i==0) {
replaceMapEntries.put("poNumber", "123");
         }
         else if (i==1) {
replaceMapEntries.put("MOID", "abc");
}}
i++;
}
}
                  try {
                  TemplateElement temp=template.getRootTreeNode();
  temp.getDescription();
  for(int i=0;i<=temp.getChildCount()-1;i++) {
  if(0!=(i%2))
  {
  String nodecheck=temp.getChildNodes().get(i).toString();
  String nodevalue=nodecheck.substring(2,nodecheck.length()-1);
if(!replaceMapEntries.containsKey(nodevalue))
  replaceMapEntries.put(nodevalue,nodecheck);
  System.out.println(nodevalue);

  }
  }
  Set s=replaceMapEntries.keySet();
  if(s.iterator().hasNext()) {
  System.out.println("hhhhhhhhhhhhhhhaaaaaaaaaaaaaaiiiiiiiiiii");
  System.out.println(s);
  }
                  writeOperation(file, template, replaceMapEntries);

}
catch(Exception e) {
configuration.setTemplateExceptionHandler(TemplateExceptionHandler.IGNORE_HANDLER);
    configuration.setWhitespaceStripping(true);
}
}

catch (Exception e) {
        RETHROW_HANDLER.getClass();

}
}


public static void writeOperation(File file, Template template, Map<String, Object> replaceMapEntries)
throws IOException, TemplateException {
Writer fileWriter = new FileWriter(file);
try {
template.process(replaceMapEntries, fileWriter);
}
catch (Exception e) {
// TODO: handle exception
         RETHROW_HANDLER.getClass();
}
FileUtils.readFileToString(file, "UTF-8");

fileWriter.flush();
fileWriter.close();
}

}

******************************************************************

for creating shortcut
https://unix.stackexchange.com/questions/388298/how-to-create-a-shortcut-for-eclipse
for adding key
https://stash.ghx.com:7893/plugins/servlet/ssh/account/keys
command for linux prodcing key:
ssh-keygen -t rsa
cd ~/.ssh/
ll
cat id_rsa.pub
use this command to build the key in Ubuntu
$ git config --global user.name "John Doe"
$ git config --global user.email johndoe@example.com
svijayakumar@ghx.com
Saravankumar Vijayakumar

system monitor is task manager here
here terminal(command prompt) itself bash

maven settings.xml change it to ghx settings.xml

for set classpath in ubuntu
https://www.ntu.edu.sg/home/ehchua/programming/howto/Environment_Variables.html

_______________________________________________________________________________

liquibase:

liquibase --defaultsFile=src/main/resources/liquibase/liquibase.properties \

--classpath="d:\mysql-connector-java-5.1.6.jar;D:\spring-learning\liquibase-helloworld-demo\target\liquibase-helloworld-demo.jar" \

--changeLogFile=liquibase/db-changelog-master.xml \

update



/home/sys-user/mysql-connector-java-5.1.6jar

liquibase --defaultsFile=src\main\resources\liquibase\liquibase.properties \

--classpath="\home\sys-user\mysql-connector-java-5.1.6jar;\home\sys-user\eclipse-workspace\corex\liquibaseProject\target\liquibase.jar\

--changeLogFile=liquibase/db-changelog-master.xml \

update



https://qxf2.com/blog/mysql-and-liquibase/

mercury like git

https://github.com/Inkimar/liquibase-example

http://shengwangi.blogspot.in/2016/04/liquibase-helloworld-example.html

internal-TestFW-CXTF-1147-UI-Heimdall-ELB-1804473841.us-west-2.elb.amazonaws.com


_____________________________________________________________________________________________________

Imperative programming

Imperative programming is a paradigm of computer programming in which the program describes a sequence of steps that change the state of the computer. Unlike declarative programming, which describes "what" a program should accomplish, imperative programming explicitly tells the computer "how" to accomplish it. Programs written this way often compile to binary executables that run more efficiently since all CPU instructions are themselves imperative statements.

To make programs simpler for a human to read and write, imperative statements can be grouped into sections known as code blocks. In the 1950s, the idea of grouping a program's code into blocks was first implemented in the ALGOL programming language. They were originally called "compound statements," but today these blocks of code are known as procedures. Once a procedure is defined, it can be used as a single imperative statement, abstracting the control flow of a program and allowing the developer to express programming ideas more naturally. This type of imperative programming is called procedural programming, and it is a step towards higher-level abstractions such as declarative programming.

Imperative programming languages

Derived from the Greek word paradeiknyai, Paradigm commonly refers to a new method of thinking about a problem or situation.

The knowledge that the Earth is round is a paradigm. Licensed from iStockPhoto. noun. The definition of a paradigm is a widely accepted example, belief or concept. An example of paradigm is evolution.

In computer science, imperative programming is a programming paradigm that uses statements that change a program's state. In much the same way that the imperative mood in natural languages expresses commands, an imperative program consists of commands for the computer to perform.


Definition: An imperative language uses a sequence of statements to determine how to reach a certain goal. These statements are said to change the state of the program as each one is executed in turn.

Examples: Java is an imperative language. For example, a program can be created to add a series of numbers:

 int total = 0;
 int number1 = 5;
 int number2 = 10;
 int number3 = 15;
 total = number1 + number2 + number3; 

Each statement changes the state of the program, from assigning values to each variable to the final addition of those values. Using a sequence of five statements the program is explicitly told how to add the numbers 5, 10 and 15 together.


Procedural and declarative programming


want to preface this by saying if you are just beginning to learn about programming it isn't important to know about imperative programming for right now.

Does that means that there is a class that it has one method that do lots of works?

It means it has statements which tell it step by step how to do the task you want it to do.

Imperative programming refers to the "type" (paradigm) of the language. In this context, imperative means we write programming statements for "how" to do it. For example, I want to calculate the total cost for buying things at the store. How do i do that? We'll 1st step is to do this, 2nd step is to do that, then etc...

public static void main(String args[]){
  // 1st step is to calculate total price of all items.
  // 2nd step is to multiple by sales tax percentage.
  // 3rd step is to add sales tax to subtotal.
  // 4th step is the return total.
}

Java is imperative, but it is also object-oriented, but it is also functional. In more advanced lessons you might learn about how Java also has other functional (paradigms) by using "lambdas".

Examples of other programming "types" (paradigms): Haskell is functional Ruby is partly functional Python is partly functional Javascript is partly functional

More information/reading:

https://en.wikipedia.org/wiki/Comparison_of_programming_paradigms

http://cs.lmu.edu/~ray/notes/paradigms/

https://en.wikipedia.org/wiki/Imperative_programming


String str = "foo";
List<String> strings = Arrays.asList("foo", "foo", "foo", "foo", "foo");
boolean allMatch = strings.stream().allMatch(s -> s.equals(str));

For Java 7 replace the last line with:

boolean allMatch = true;
for (String string : strings) {
    if (!string.equals(str)) {
        allMatch = false;
        break;
    }
}
*********************************************************
package com.ghx.testframework.action.find_and_replace;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;

import org.apache.commons.io.FileUtils;

import com.ghx.common.log.GHXLogger;
import com.ghx.common.log.GHXLoggerFactory;
import com.ghx.core.domain.DocumentContext;
import com.ghx.tpm.api.internalization.annotations.Internalization;
import com.ghx.uiconfig.annotation.UIConfigurable;
import com.ghx.uiconfig.enums.ComponentType;

import freemarker.cache.FileTemplateLoader;
import freemarker.core.TemplateElement;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import lombok.Data;

@Data
@Internalization(bundleName = "i18n.FindAndReplace", moduleName = "FindAndReplace")
@UIConfigurable(value = ComponentType.testframework, displayName = "com.ghx.label.findandreplace.display.name", description = "com.ghx.label.findandreplace.description")


public class findAndReplaceCheck {

public static void main(String[] args) {
// TODO Auto-generated method stub
// TODO Auto-generated method stub
DocumentContext docContext = new DocumentContext();
        docContext.addItem(DocumentContext.Key.DOCUMENT_INPUT_FILENAME, "/home/sys-user/backup/Desktop/multi.edi"); 
docContext.addItem(DocumentContext.Key.OUTBOUND_FILE_NAME, "/home/sys-user/backup/Desktop/multi.edi");
final GHXLogger LOGGER = GHXLoggerFactory.getLogger(FindAndReplace.class);

if (docContext == null) {
System.out.println("Input file name parameter value null");
}
if (!docContext.contains(DocumentContext.Key.DOCUMENT_INPUT_FILENAME)) {
System.out.println("Input file name parameter value Empty");
}
String inputFileName = docContext.getItem(DocumentContext.Key.DOCUMENT_INPUT_FILENAME);
LOGGER.info("Started performing find and replace action for input file name {}", inputFileName);
Configuration configuration = new Configuration(Configuration.VERSION_2_3_23);
configuration.setTemplateExceptionHandler(TemplateExceptionHandler.IGNORE_HANDLER);
configuration.setWhitespaceStripping(true);
configuration.setLocalizedLookup(false);
File file = new File(inputFileName);
try {
FileTemplateLoader fileTemplateLoader = new FileTemplateLoader(new File(file.getParent() + File.separator));
configuration.setTemplateLoader(fileTemplateLoader);
Template template = configuration.getTemplate(file.getName());
Map<String, Object> replaceMapEntries = new HashMap<>();
LOGGER.debug("Find and replace entry...{} with {}","poNumber",
"adas");
String replaceText = "asada";
replaceMapEntries.put("poNumber", "adas");
writeOperation(file, template, replaceMapEntries);
} catch (IOException | TemplateException e) {
LOGGER.error("Write Operation failed when trying to find and replace.", e);
System.out.println("Write Operation failed when trying to find and replace.");
}
}
private static String writeOperation(File file, Template template, Map<String, Object> replaceMapEntries)
throws TemplateException, IOException {
TemplateElement temp = template.getRootTreeNode();
temp.getDescription();
for (int i = 0; i <= temp.getChildCount() - 1; i++) {
if (0 != (i % 2)) {
String nodecheck = temp.getChildNodes().get(i).toString();
String nodevalue = nodecheck.substring(2, nodecheck.length() - 1);
if (!replaceMapEntries.containsKey(nodevalue))
replaceMapEntries.put(nodevalue, nodecheck);
}
}
Writer fileWriter = new FileWriter(file);
template.process(replaceMapEntries, fileWriter);
fileWriter.flush();
fileWriter.close();
return FileUtils.readFileToString(file, StandardCharsets.UTF_8);
}

private String generateRandomString(Random random, int length) {
return random.ints(48, 122).filter(i -> (i < 57 || i > 65) && (i < 90 || i > 97)).mapToObj(i -> (char) i)
.limit(length).collect(StringBuilder::new, StringBuilder::append, StringBuilder::append).toString();
}
}
___________________________________________

FindAndReplaceTwo.java

package com.ghx.testframework.action.find_and_replace;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;

import javax.swing.tree.TreePath;

import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.util.TypeKey;
import com.ghx.common.log.GHXLogger;
import com.ghx.common.log.GHXLoggerFactory;
import com.ghx.core.domain.DocumentContext;
import com.ghx.testframework.action.TestFrameworkAction;
import com.ghx.testframework.action.change_control_number.panel.SegmentNamePanel;
import com.ghx.testframework.utils.exception.TestFWServerException;
import com.ghx.tpm.api.impl.internalization.InternalizatorFactory;
import com.ghx.tpm.api.internalization.IInternalizator;
import com.ghx.tpm.api.internalization.annotations.Internalization;
import com.ghx.uiconfig.annotation.UIConfigurable;
import com.ghx.uiconfig.annotation.UIDiv;
import com.ghx.uiconfig.annotation.UIField;
import com.ghx.uiconfig.annotation.populator.UIFieldModelPopulator;
import com.ghx.uiconfig.enums.ComponentType;
import com.ghx.uiconfig.enums.FieldType;

import antlr.StringUtils;
import bsh.StringUtil;
import freemarker.cache.FileTemplateLoader;
import freemarker.core.Environment;
import freemarker.core.InvalidReferenceException;
import freemarker.core.TemplateClassResolver;
import freemarker.core.TemplateElement;
import freemarker.template.Configuration;
import freemarker.template.DefaultObjectWrapper;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import freemarker.template.TemplateModel;
import freemarker.template.TemplateSequenceModel;
import lombok.Data;

@Data
@Internalization(bundleName = "i18n.FindAndReplace", moduleName = "FindAndReplace")
@UIConfigurable(value = ComponentType.testframework, displayName = "com.ghx.label.findandreplace.display.name", description = "com.ghx.label.findandreplace.description")
public class FindAndReplaceTwo implements TestFrameworkAction ,TemplateExceptionHandler{

private static final IInternalizator INTERNALIZATOR = InternalizatorFactory.getInternalizator(FindAndReplaceTwo.class);
private static final GHXLogger LOGGER = GHXLoggerFactory.getLogger(FindAndReplaceTwo.class);
private static final String CUSTOM_ACTION = "Custom String";

@UIField(type = FieldType.multipleDynamicKeyValue, order = 1, label = "com.ghx.label.findandreplace.findandreplaceentries", panel = @UIDiv(value = SegmentNamePanel.class))
@UIFieldModelPopulator(FindAndReplaceEntryPopulator.class)
private static List<FindAndReplaceEntry> findAndReplaceEntries;

@UIField(type = FieldType.checkbox, label = "com.ghx.label.findandreplace.failonerror", order = 2)
private boolean failOnError;


public static void main(String args[]) {
// TODO Auto-generated method stub


try {
List<FindAndReplaceEntry> findAndReplace = new ArrayList<FindAndReplaceEntry>();

DocumentContext docContext = new DocumentContext();
        docContext.addItem(DocumentContext.Key.DOCUMENT_INPUT_FILENAME, "/home/sys-user/Desktop/testDocument.edi"); 
docContext.addItem(DocumentContext.Key.OUTBOUND_FILE_NAME, "/home/sys-user/Desktop/testDocument.edi");

if (docContext != null && docContext.contains(DocumentContext.Key.DOCUMENT_INPUT_FILENAME)) {

String inputFileName = docContext.getItem(DocumentContext.Key.DOCUMENT_INPUT_FILENAME);
LOGGER.info("Started performing find and replace action for input file name {}", inputFileName);
//Find_Replace_DOCX.modifyFile(inputFileName, "MOID", "12211");

File file = new File(inputFileName);
File file1 = new File("testDocument.edi");

Configuration configuration = new Configuration(Configuration.VERSION_2_3_20);
configuration.setLocalizedLookup(false);
configuration .setTemplateExceptionHandler(TemplateExceptionHandler.IGNORE_HANDLER);
/*configuration
            .setNewBuiltinClassResolver(TemplateClassResolver.UNRESTRICTED_RESOLVER);*/
FileTemplateLoader fileTemplateLoader = new FileTemplateLoader(
new File(file.getParent() + File.separator));
configuration.setTemplateLoader(fileTemplateLoader);
configuration.setObjectWrapper(new DefaultObjectWrapper());
Template template = configuration.getTemplate(file.getName());
FindAndReplaceEntry f1 = new FindAndReplaceEntry("poNumber", "123", null);
FindAndReplaceEntry f2 = new FindAndReplaceEntry("MOID", "abc", null);
findAndReplace.add(f1);
findAndReplace.add(f2);
Map<String, Object> replaceMapEntries = new HashMap<>();

                  for(int i=0;i<findAndReplace.size();) {
FindAndReplaceEntry findAndReplaceEntry =findAndReplace.get(i) ;

LOGGER.info("Find and replace entry...{}", findAndReplaceEntry);
if (findAndReplaceEntry != null) {
String action = CUSTOM_ACTION;
if (action != null && action.equalsIgnoreCase(CUSTOM_ACTION)) {
         if (i==0) {
replaceMapEntries.put("poNumber", "123");
         }
         else if (i==1) {
replaceMapEntries.put("MOID", "abc");
}}
i++;
}
}
                
/*String node=temp.getChildNodes().get(5).toString();
String node1=node.substring(2,node.length()-1);
System.out.println(node1);
                  replaceMapEntries.put(node1,node);*/
                  try {
/*TemplateElement temp=template.getRootTreeNode();
temp.getDescription();
String node=temp.getChildNodes().get(5).toString();*/
/* System.out.println(node);
*/ /*for (int i=0 ;i<=node;i++) {
System.out.println(temp.getChildNodes());

}*/
                  TemplateElement temp=template.getRootTreeNode();
  temp.getDescription();
  for(int i=0;i<=temp.getChildCount()-1;i++) {
  if(0!=(i%2))
  {
  String nodecheck=temp.getChildNodes().get(i).toString();
  String nodevalue=nodecheck.substring(2,nodecheck.length()-1);
if(!replaceMapEntries.containsKey(nodevalue))
  replaceMapEntries.put(nodevalue,nodecheck);
  System.out.println(nodevalue);

  }
  }
  Set s=replaceMapEntries.keySet();
  if(s.iterator().hasNext()) {
  System.out.println("hhhhhhhhhhhhhhhaaaaaaaaaaaaaaiiiiiiiiiii");
  System.out.println(s);
  }
                  writeOperation(file, template, replaceMapEntries);

}
catch(Exception e) {
configuration.setTemplateExceptionHandler(TemplateExceptionHandler.IGNORE_HANDLER);
    configuration.setWhitespaceStripping(true);
}
}

catch (Exception e) {
        RETHROW_HANDLER.getClass();

}
}


public static void writeOperation(File file, Template template, Map<String, Object> replaceMapEntries)
throws IOException, TemplateException {
Writer fileWriter = new FileWriter(file);
try {
template.process(replaceMapEntries, fileWriter);
}
catch (Exception e) {
// TODO: handle exception
         RETHROW_HANDLER.getClass();
}
FileUtils.readFileToString(file, "UTF-8");

fileWriter.flush();
fileWriter.close();
}



@Override
public String apply(DocumentContext docContext, String bodyToChange, List<String> newValue)
throws TestFWServerException {
// TODO Auto-generated method stub
return null;
}


/*@Override
public void handleTemplateException(TemplateException arg0, Environment arg1, Writer arg2)
throws TemplateException {
// TODO Auto-generated method stub
       RETHROW_HANDLER.getClass();
       Configuration configuration = new Configuration();
       configuration
       .setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
}
*/
}
/*class MyTemplateExceptionHandler implements TemplateExceptionHandler {
    public void handleTemplateException(TemplateException te, Environment env, java.io.Writer out)
            throws TemplateException {
        RETHROW_HANDLER.handleTemplateException(te, env, out);
    }
}*/

___________________________________________________________________________________

Find_replace_DOCX.java

package com.ghx.testframework.action.find_and_replace;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
 
public class Find_Replace_DOCX
{    
    static void modifyFile(String filePath, String oldString, String newString)
    {
        File fileToBeModified = new File(filePath);
         
        String oldContent = "";
         
        BufferedReader reader = null;
         
        FileWriter writer = null;
         
        try
        {
            reader = new BufferedReader(new FileReader(fileToBeModified));
             
            //Reading all the lines of input text file into oldContent
             
            String line = reader.readLine();
             
            while (line != null) 
            {


              /*  \r is the return character (0x0D)

                You could use it when you open a file in binary mode but still want to add the line \r\n line termination. 
*/

                oldContent = oldContent + line + System.lineSeparator();
                 
                line = reader.readLine();
            }
             
            //Replacing oldString with newString in the oldContent
             
            String newContent = oldContent.replaceAll(oldString, newString);
             
            //Rewriting the input text file with newContent
             
            writer = new FileWriter(fileToBeModified);
             
            writer.write(newContent);
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        finally
        {
            try
            {
                //Closing the resources
                 
                reader.close();
                 
                writer.close();
            } 
            catch (IOException e) 
            {
                e.printStackTrace();
            }
        }
    }
     
    public static void main(String[] args)
    {
        modifyFile("/home/sys-user/Desktop/test_find_850_1.edi", "poNumber", "qwerty");
        modifyFile("/home/sys-user/Desktop/test_find_850_1.edi", "MOID", "abc");
        System.out.println("done");
    }
}

***************************************************

PYTHON GRAPH PLOT:

import numpy as np
import matplotlib.pyplot as plt

# data to plot
n_groups = 4
means_Resolved = (90, 55, 40, 65)
means_Success = (85, 62, 54, 20)
means_Inprogress = (70, 12, 50, 80)
means_Error = (10,79,32,55)

# create plot
fig, ax = plt.subplots()
index = np.arange(n_groups)
bar_width = 0.10
opacity = 0.8

rects1 = plt.bar(index, means_Resolved, bar_width,
alpha=opacity,
color='b',
label='Resolved')

rects2 = plt.bar(index + bar_width, means_Success, bar_width,
alpha=opacity,
color='g',
label='Success')

rects3 = plt.bar(index + bar_width+bar_width, means_Inprogress, bar_width,
alpha=opacity,
color='y',
label='Inprogress')

rects3 = plt.bar(index + bar_width+bar_width+bar_width, means_Error,\
         bar_width,
alpha=opacity,
color='r',
label='Error')

plt.xlabel('Doc Flow Comparision rate')
plt.ylabel('Scores')
plt.title('Scores by each test Run')
plt.xticks(index + bar_width, ('Test1', 'Test2', 'Test3', 'Test4'))
plt.legend()

plt.tight_layout()
plt.show()

***********************************************

cleanStack.py:

""" This Lambda function is to delete instance of TFW service older than max day """
import os
import datetime
import calendar
import logging
import boto3

LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)
CHARSET = 'UTF-8'
def lambda_handler():
    """ This handler is of python2.7 lambda function, we have weekly trigger too"""
    stack_for_deletion = []
    cloudformation = boto3.resource('cloudformation')
    client = boto3.client('cloudformation')
    status_to_be_deleted = os.environ.get("status_to_be_deleted")
    filter_service = os.environ.get("filter_service")
    max_holding_days = os.environ.get("max_holding_days")
    deletion_per_week_on = os.environ.get("deletion_per_week_on")
    filter_for_deletion = os.environ.get("filter_for_deletion").split(",")
    LOGGER.info('This is an info message')
    LOGGER.info("Max_Holding_Days: %s", max_holding_days)
    LOGGER.info("For Max holding days ,Instance tag with Name Containing: %s"\
          , filter_for_deletion)
    for stack in cloudformation.stacks.all():
        test_framework_stacks = [stack for s in filter_for_deletion if s.lower() \
        in stack.name.lower()]
        stacks = [stack.name for stack in test_framework_stacks if stack.stack_status \
                  in status_to_be_deleted]
        for test_framework_stack in test_framework_stacks:
            for tag in test_framework_stack.tags:
                if (tag['Key'] == 'service') and (tag['Value'] == filter_service):
                    if stacks:
                        stack_for_deletion.append(stacks)
                    if calendar.day_name[datetime.datetime.today().weekday()]\
                    == deletion_per_week_on:
                        stack_for_deletion.append(test_framework_stack.name)
                    stack_creation_date = datetime.datetime.strptime(\
                    str(test_framework_stack.creation_time).split(".")[0], '%Y-%m-%d %H:%M:%S')
                    current_time = datetime.datetime.strptime(str(\
                            datetime.datetime.now()).split(".")[0],\
                            '%Y-%m-%d %H:%M:%S')
                    diff_time = current_time-stack_creation_date
                    if diff_time.days > float(max_holding_days):
                        stack_for_deletion.append(test_framework_stack.name)
    for stack_to_delete in stack_for_deletion:
        response = client.delete_stack(
            StackName=stack_to_delete
        )
        LOGGER.info("Going to delete: %s", response)

____________________________________
""" This Lambda function is to delete instance of TFW service older than max day """
import os
import traceback
import datetime
from datetime import date
import calendar
import logging
import boto3
from botocore.exceptions import ClientError
from dateutil.parser import parse

LOGGER = logging.getLogger()
LOGGER.setLevel(logging.INFO)
CHARSET = 'UTF-8'

def delete_instance_handler(event, context):
    """ This handler is of python2.7 lambda function, we have weekly trigger too"""
    ec2 = boto3.resource('ec2')
    filter_service = os.environ.get("filter_service")
    max_holding_days = os.environ.get("max_holding_days")
    deletion_per_week_on = os.environ.get("deletion_per_week_on")
    filter_for_max_days = os.environ.get("filter_for_max_days").split(",")
    filter_for_weekly_deletion = os.environ.get("filter_for_weekly_deletion").split(",")
    test_framework_maintainer = os.environ.get("test_framework_maintainer").split(",")
    LOGGER.info('This is an info message')
    LOGGER.info("Max_Holding_Days: %s", max_holding_days)
    LOGGER.info("For Max holding days ,Instance tag with Name Containing: %s"\
          , filter_for_max_days)
    LOGGER.info("For Weekly deletion ,Instance tag with Name Containing: %s"\
          , filter_for_weekly_deletion)
    i = 0
    while i < len(filter_for_max_days):
        filter_for_max_days[i] = "*"+filter_for_max_days[i]+"*"
        i += 1
    i = 0
    while i < len(filter_for_weekly_deletion):
        filter_for_weekly_deletion[i] = "*"+filter_for_weekly_deletion[i]+"*"
        i += 1
    instance_for_weekly_deletion = ec2.instances.filter(Filters=[{'Name': 'tag:Name',\
        'Values': filter_for_weekly_deletion},\
        {'Name': 'tag:service', 'Values': [filter_service]}])
    instance_deletion = []
    for instance in instance_for_weekly_deletion:
        if calendar.day_name[datetime.datetime.today().weekday()] == deletion_per_week_on:
            LOGGER.info("Today is Saturday, Going to delete instances %s", instance.id)
            instance_deletion.append(instance.id)
    instance_for_max_days = ec2.instances.filter(Filters=[{'Name': 'tag:Name',\
        'Values': filter_for_max_days},\
        {'Name': 'tag:service', 'Values': [filter_service]}])
    for instance in instance_for_max_days:
        launch_time_today_difference = (parse(date.isoformat(date.today())))-(parse(date\
                                       .isoformat(instance.launch_time)))
        if launch_time_today_difference.days > int(max_holding_days):
            LOGGER.info("Going beyond max holding day: %s, Going to delete instances: %s",\
                      max_holding_days, instance.id)
            instance_deletion.append(instance.id)
    LOGGER.info('*************************************************************')
    LOGGER.info("The Instances that will be deleted are: %s", instance_deletion)
    LOGGER.info("Total: %s", len(instance_deletion))
    if instance_deletion:
        ec2.instances.filter(InstanceIds=instance_deletion).terminate()
        send_mail("ec2-alerts@ghx.com", test_framework_maintainer,\
        "Test Framework Ec2 Instance Maintenance Update",\
        "The Instances that will be deleted are:" + "\n".join(instance_deletion))
    else:
        send_mail("ec2-alerts@ghx.com", test_framework_maintainer,\
        "Test Framework Ec2 Instance Maintenance Update",\
        "No Instances deleted." + "\n".join(instance_deletion))
    LOGGER.info('*************************************************************')

def send_mail(sender, recipients, subject, result_msg):
    """ This part of code is to send mail to maintainer"""
    try:
        client = boto3.client('ses', 'us-east-1')
        client.send_email(Destination={'ToAddresses': recipients},\
        Message={'Body': {'Text': {'Charset': CHARSET, 'Data': result_msg}},\
        'Subject': {'Charset': CHARSET, 'Data': subject}}, Source=sender)
    except ClientError as exception:
        LOGGER.info(exception)
        traceback.print_exc()


******************************************************************************

eclipse arguement:


-Dghx.configsvc.db.url=mongodb://configsvc-user:configsvc123@localhost:27017/configsvc
-Dghx.engine.id=UI-Heimdall
-Xms512m -Xmx2048m -XX:+UseParallelGC -XX:PermSize=256M -XX:MaxPermSize=2048m
-Dautomatic.user.approvel=true
-Dlogback.configurationFile=logback-LOCAL.xml
-Dliquibase.jdbc.user=egxapp
-Dliquibase.jdbc.password=password
-Dliquibase.jdbc.url=jdbc:mysql://localhost:3306/tpm
-Dliquibase.onerror.fail=false
-Dauth.mode=local
-Dliquibase.contexts=LOCAL
-Dliquibase.shouldRun=true
-Daws_access_key_id=ASIAQXL43UGU2GQUAX6Z
-Daws_secret_access_key=muXOibUQxEjACNm+IR7m0XcYUk5m2FKIDRjpcrSk

Increase Eclipse (eclipse-ini):

-Xms256m
-Xmx4096m

_______________________________________________________________________

Postman used for rest call check (snapshot):

jspdf, Dom-to-image, 


Maven Cobertura code coverage
build user project with :   mvn cobertura:cobertura

got to target of the project and check site folder and open coburtura and click index.html you will get UI of code coverage. you can also get code coverage  with eclipse and some junit plugins.

  <dependency>
            <groupId>org.powermock</groupId>
            <artifactId>powermock-api-support</artifactId>
        </dependency>
        <dependency>
            <groupId>org.powermock</groupId>
            <artifactId>powermock-module-junit4</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.powermock</groupId>
            <artifactId>powermock-api-mockito</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.ghx.rules</groupId>
            <artifactId>rules-egx-test</artifactId>
            <scope>test</scope>
        </dependency>

**************************************************

<reporting>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<version>RELEASE</version>
<reportSets>
<reportSet>
<reports>
<report>checkstyle</report>
</reports>
</reportSet>
</reportSets>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-pmd-plugin</artifactId>
<version>RELEASE</version>
<configuration>
<rulesets>
<ruleset>/rulesets/favorites.xml</ruleset>
</rulesets>
<targetJdk>1.8</targetJdk>
</configuration>
</plugin>
</plugins>
</reporting>

<build>
<plugins>
<!-- Code coverage plugin to produce test reports -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
<version>2.7</version>
<configuration>
<instrumentation>
<ignoreMethodAnnotations>
<ignoreMethodAnnotation>com.ghx.testframework.api.dao</ignoreMethodAnnotation>
</ignoreMethodAnnotations>
<ignoreMethodAnnotation>com.thermofisher.micro.common.annotation.CoberturaIgnore</ignoreMethodAnnotation>
<ignoreMethodAnnotations>
<ignoreMethodAnnotation>lombok.Generated</ignoreMethodAnnotation>
<ignoreMethodAnnotation>javax.annotation.Generated</ignoreMethodAnnotation>
<ignoreMethodAnnotation>org.springframework.stereotype.Repository</ignoreMethodAnnotation>
</ignoreMethodAnnotations>
<ignoreTrivial>true</ignoreTrivial>
<ignores>
<ignore>com.ghx.testframework.action.change_control_number.ChangeControlNumberPV4PSegmentNamePanelWO1WVSegmentNameValidatorRestAPI</ignore>
<ignore>com.ghx.testframework.action.find_and_replace.FindAndReplaceEntryPopulator</ignore>
<ignore>com.ghx.testframework.api.entity.ScenarioPopulationRestAPI</ignore>
<ignore>com.ghx.testframework.api.entity.ScenarioValidationRestAPI</ignore>
<ignore>com.ghx.testframework.api.entity.ConnectionPopulationRestAPI</ignore>
<ignore>com.ghx.testframework.api.entity.ConnectionValidationRestAPI</ignore>
<ignore>org.springframework.stereotype.Repository</ignore>

</ignores>
<excludes>
<exclude>com/ghx/testframework/action/change_control_number/ChangeControlNumberPV4PSegmentNamePanelWO1WVSegmentNameValidatorRestAPI.class</exclude>
<exclude>com/ghx/testframework/action/find_and_replace/FindAndReplaceEntryPopulator.class</exclude>
<exclude>com/ghx/testframework/api/entity/ScenarioPopulationRestAPI.class</exclude>
<exclude>com/ghx/testframework/api/entity/ScenarioValidationRestAPI.class</exclude>
<exclude>com/ghx/testframework/api/entity/ConnectionPopulationRestAPI.class</exclude>
<exclude>com/ghx/testframework/api/entity/ConnectionValidationRestAPI.class</exclude>
<exclude>org/springframework/stereotype/*</exclude>


</excludes>
</instrumentation>
</configuration>
<executions>
<execution>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

lombok.config:
lombok.addJavaxGeneratedAnnotation = true
lombok.addLombokGeneratedAnnotation = true

***************************************************

tfw-gru-cf-template.json:

{
    "AWSTemplateFormatVersion": "2010-09-09",
    "Description": "Test Framework Gru stack",
    "Parameters": {
        "VPCStackName": {
            "Description": "Name of the VPC CloudFormaiton Stack",
            "Type": "String",
            "Default": "LegacyVpcCfOutputs"
        },
        "InstanceType": {
            "Description": "WebServer EC2 instance type",
            "Type": "String",
            "Default": "m4.large",
            "AllowedValues": [
                "m3.medium",
                "m3.large",
                "m4.large",
                "m4.xlarge",
                "m4.2xlarge",
                "m5a.large",
                "m5a.xlarge",
                "t3a.medium",
                "t3a.small",
                "t2.medium",
                "t2.small"
            ],
            "ConstraintDescription": "must be a valid EC2 instance type."
        },
        "KeyName": {
            "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
            "Type": "AWS::EC2::KeyPair::KeyName",
            "ConstraintDescription": "must be the name of an existing EC2 KeyPair."
        },
        "configSVCurl": {
            "Description": "The URL of the configsvc to use",
            "Type": "String",
            "ConstraintDescription": "must be the name of an existing configsvc mongo database."
        },
        "configSVCpass": {
            "Description": "The encrypted password of the configsvc to use",
            "Type": "String",
            "ConstraintDescription": "must be the password of an existing configsvc mongo database."
        },
        "AdditionalJavaOpts": {
            "Description": "Additional JAVA_OPTS as system properties, -Dname=value",
            "Type": "String",
            "Default": "",
            "ConstraintDescription": "Set as system property, -D"
        },
        "ScalingMax": {
            "Description": "Max Number to scale EC2 instances to",
            "Type": "Number",
            "Default": "1"
        },
        "ScalingMin": {
            "Description": "Min Number to scale EC2 instances to",
            "Type": "Number",
            "Default": "1"
        },
        "GHXEnvironment": {
            "Description": "GHX environment",
            "Type": "String",
            "Default": "test-framework-gru"
        },
        "TagName": {
            "Description": "Name of EC2 instances launched from AutoScalingGroup",
            "Type": "String"
        },
        "Service": {
            "Description": "Value to apply to the service tag for instances",
            "Type": "String",
            "Default": "test-framework-gru"
        },
        "InstanceMonitoring": {
            "Description": "Set to false for Basic monitoring and true for Detailed monitoring",
            "Type": "String",
            "Default": "false"
        },
        "Application": {
            "Description": "Application   To Map To Application ",
            "Type": "String",
            "ConstraintDescription": "Must Be Valid Application Label"
        },
        "ApplicationVersion": {
            "Description": "Application Version Number To Map To Application Key In S3",
            "Type": "String",
            "ConstraintDescription": "Must Be Valid Application Version Label"
        },
        "BranchName": {
            "Description": "Bamboo Branch Name",
            "Type": "String",
            "ConstraintDescription": "Must be valid branch name"
        },
        "BuildNumber": {
            "Description": "Bamboo build number",
            "Type": "String",
            "ConstraintDescription": "must be a valid bamboo build number"
        },
        "War": {
            "Description": "The War Name",
            "Type": "String",
            "ConstraintDescription": "Name of the war to deployed"
        },
        "DeploymentBucket": {
            "Description": "Set to true to create SNS notifications for instance_terminations of ASGs",
            "Type": "String",
            "Default": "ghx-deployment"
        },
        "ELBStackName": {
            "Description": "Name of the ELB Stack for this component",
            "Type": "String"
        },
        "Product":{
            "Description":"Product name. Used to determine asset locations in deployment bucket",
            "Type":"String",
            "Default": "corex"
        },
        "NewRelicEnabled": {
            "Description": "Select true to enable newrelic monitoring of this system",
            "Type": "String",
            "Default": "false",
            "AllowedValues": [
                "true",
                "false"
            ],
            "ConstraintDescription": "must be true or false"
        },
        "NewRelicInfraEnabled": {
            "Description": "Select true to enable newrelic infra monitoring of this system",
            "Type": "String",
            "Default": "false",
            "AllowedValues": [
                "true",
                "false"
            ],
            "ConstraintDescription": "must be true or false"
        }
    },

    "Mappings": {
        "AWSRegionArch2AMI": {
            "us-east-1": {
                "AMIID": "TOKENGRUAMIUSEAST1"
            },

            "us-west-2": {
                "AMIID": "TOKENGRUAMIUSWEST2"
            }
        },

        "Ec2Role": {
            "CUSTOM": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },

            "DEV": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },

            "STG": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },

            "DEVINT": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },

            "CoreXBINT": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },

            "PRD": {
                "Role": "arn:aws:iam::326626545197:instance-profile/DevOpsRole-test-framework"
            },

            "LOAD": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            }
        }

    },

    "Resources": {
        "NetworkInfo": {
            "Type": "Custom::NetworkInfo",
            "Properties": {
                "ServiceToken": {
                    "Fn::Join": [
                        "", [
                            "arn:aws:lambda:", {
                                "Ref": "AWS::Region"
                            },

                            ":", {
                                "Ref": "AWS::AccountId"
                            },

                            ":function:DescribeCFStackOutputs"
                        ]
                    ]
                },

                "StackName": {
                    "Ref": "VPCStackName"
                }
            }
        },
        "ELBInfo": {
            "Type": "Custom::ELBInfo",
            "Properties": {
                "ServiceToken": {
                    "Fn::Join": ["", [
                        "arn:aws:lambda:",
                        {
                            "Ref": "AWS::Region"
                        }, ":", {
                            "Ref": "AWS::AccountId"
                        },
                        ":function:DescribeCFStackOutputs"
                    ]]
                },
                "StackName": {
                    "Ref": "ELBStackName"
                }
            }
        },
        "MercuryGruInstanceSecurityGroup": {
            "Type": "AWS::EC2::SecurityGroup",
            "Properties": {
                "GroupDescription": "Security Group for Gru Stack EC2 Instances",
                "SecurityGroupEgress": [{
                    "CidrIp": "0.0.0.0/0",
                    "FromPort": "0",
                    "IpProtocol": "-1",
                    "ToPort": "65535"
                }],

                "SecurityGroupIngress": [{
                    "CidrIp": "0.0.0.0/0",
                    "FromPort": "0",
                    "IpProtocol": "-1",
                    "ToPort": "65535"
                }],

                "VpcId": {
                    "Fn::GetAtt": [
                        "NetworkInfo",
                        "VpcId"
                    ]
                }
            }
        },

        "MercuryGruLaunchConfig": {
            "Type": "AWS::AutoScaling::LaunchConfiguration",
            "Metadata": {
                "AWS::CloudFormation::Init": {
                    "configSets": {
                        "setupall": [
                            "paramfile",
                            "logstash",
                            "cpumetriccollection",
                            "deploy",
                            "newrelic"
                        ]
                    },
                    "paramfile": {
                        "files": {
                            "/opt/conf/cf-params.txt": {
                                "content": {
                                    "Fn::Join": ["", [
                                        "AdditionalJavaOpts:", {
                                            "Ref": "AdditionalJavaOpts"
                                        }, "\n",
                                        "ApplicationVersion:", {
                                            "Ref": "ApplicationVersion"
                                        }, "\n",
                                        "BranchName:", {
                                            "Ref": "BranchName"
                                        }, "\n",
                                        "BuildNumber:", {
                                            "Ref": "BuildNumber"
                                        }, "\n",
                                        "CFStackName:", {
                                            "Ref": "AWS::StackName"
                                        }, "\n",
                                        "configSVCurl:", {
                                            "Ref": "configSVCurl"
                                        }, "\n",
                                        "InstanceType:", {
                                            "Ref": "InstanceType"
                                        }, "\n",
                                        "KeyName:", {
                                            "Ref": "KeyName"
                                        }, "\n",
                                        "region:", {
                                            "Ref": "AWS::Region"
                                        }, "\n",
                                        "ScalingMax:", {
                                            "Ref": "ScalingMax"
                                        }, "\n",
                                        "ScalingMin:", {
                                            "Ref": "ScalingMin"
                                        }, "\n",
                                        "VPCStackName:", {
                                            "Ref": "VPCStackName"
                                        }, "\n",
                                        "service:", {
                                            "Ref": "Service"
                                        }, "\n",
                                        "Application:", {
                                            "Ref": "Application"
                                        }, "\n",
                                        "GHXEnvironment:", {
                                            "Ref": "GHXEnvironment"
                                        }, "\n",
                                        "War:", {
                                            "Ref": "War"
                                        }, "\n",
                                        "DeploymentBucket:", {
                                            "Ref": "DeploymentBucket"
                                        }, "\n",
                                        "Product:", {
                                            "Ref": "Product" 
                                        }, "\n",
                                        "NewRelicEnabled:", {
                                            "Ref": "NewRelicEnabled"
                                        }, "\n",
                                        "NewRelicInfraEnabled:", {
                                            "Ref": "NewRelicInfraEnabled"
                                        }, "\n"
                                    ]]
                                },
                                "mode": "000644",
                                "owner": "root",
                                "group": "root"
                            }
                        },
                        "commands": {
                            "01-createdirs": {
                                "command": "/bin/mkdir /opt/scripts; /bin/mkdir -p /opt/conf/; /bin/mkdir -p /var/log/cpumetric/;"
                            }
                        }
                    },

                    "logstash": {
                        "commands": {
                            "01-get-logstash-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            {
                                                "Ref": "ApplicationVersion"
                                            }, "/",
                                            {
                                                "Ref": "BranchName"
                                            }, "/",
                                            {
                                                "Ref": "BuildNumber"
                                            },
                                            "/cf-deploy/scripts/cf-config-logstash-jmx.py /opt/scripts/cf-config-logstash-jmx.py",
                                            " ; chmod 744 /opt/scripts/cf-config-logstash-jmx.py"
                                        ]
                                    ]
                                }
                            },
                            "05-config-logstash": {
                                "command": "/opt/scripts/cf-config-logstash-jmx.py"
                            },
                            "11-get-start-ls": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            {
                                                "Ref": "ApplicationVersion"
                                            }, "/",
                                            {
                                                "Ref": "BranchName"
                                            }, "/",
                                            {
                                                "Ref": "BuildNumber"
                                            },
                                            "/cf-deploy/scripts/start-ls.sh /opt/scripts/start-ls.sh",
                                            " ; chmod 744 /opt/scripts/start-ls.sh"
                                        ]
                                    ]
                                }
                            },
                            "15-start-logstash": {
                                "command": "/opt/scripts/start-ls.sh"
                            }
                        }
                    },

                    "newrelic": {
                        "commands": {
                            "01-get-newrelic-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://",
                                            {
                                                "Ref": "DeploymentBucket"
                                            },
                                            "/corex/",
                                            {
                                                "Ref": "ApplicationVersion"
                                            }, "/",
                                            {
                                                "Ref": "BranchName"
                                            }, "/",
                                            {
                                                "Ref": "BuildNumber"
                                            },
                                            "/cf-deploy/scripts/cf-install-newrelic.py /opt/scripts/cf-install-newrelic.py",
                                            " ; chmod 744 /opt/scripts/cf-install-newrelic.py"
                                        ]
                                    ]
                                }
                            },
                            "05-deploy-newrelic": {
                                "command": "/opt/scripts/cf-install-newrelic.py"
                            }
                        }
                    },

                    "cpumetriccollection": {
                        "commands": {
                            "01-get-testfwmetrics-cpu-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            {
                                                "Ref": "ApplicationVersion"
                                            }, "/",
                                            {
                                                "Ref": "BranchName"
                                            }, "/",
                                            {
                                                "Ref": "BuildNumber"
                                            },
                                            "/cf-deploy/scripts/cf-start-cpumetriccollection.py /opt/scripts/cf-start-cpumetriccollection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-cpumetriccollection.py"
                                        ]
                                    ]
                                }
                            },
                            "11-get-start-cpu": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            {
                                                "Ref": "ApplicationVersion"
                                            }, "/",
                                            {
                                                "Ref": "BranchName"
                                            }, "/",
                                            {
                                                "Ref": "BuildNumber"
                                            },
                                            "/cf-deploy/scripts/start-cpu.sh /opt/scripts/start-cpu.sh",
                                            " ; chmod 744 /opt/scripts/start-cpu.sh"
                                        ]
                                    ]
                                }
                            },
                            "12-get-crontab": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            {
                                                "Ref": "ApplicationVersion"
                                            }, "/",
                                            {
                                                "Ref": "BranchName"
                                            }, "/",
                                            {
                                                "Ref": "BuildNumber"
                                            },
                                            "/cf-deploy/scripts/crontab /tmp/crontab",
                                            " ; cat /tmp/crontab >> /etc/crontab"
                                        ]
                                    ]
                                }
                            },
                            "15-start-cpu": {
                                "command": "/opt/scripts/start-cpu.sh"
                            }
                        }
                    },

                    "deploy": {
                        "files": {
                            "/etc/cfn/cfn-hup.conf": {
                                "content": {
                                    "Fn::Join": [
                                        "", [
                                            "[main]\n",
                                            "stack=", {
                                                "Ref": "AWS::StackId"
                                            },

                                            "\n",
                                            "region=", {
                                                "Ref": "AWS::Region"
                                            },

                                            "\n",
                                            "interval=1\n"
                                        ]
                                    ]
                                }
                            },

                            "/var/tmp/deployScript/prep.sh": {
                                "content": {
                                    "Fn::Join": [
                                        "", [
                                            "#!/bin/bash",
                                            "\n",
                                            "date > /home/ec2-user/starttime\n",
                                            "ipaddress=`curl http://169.254.169.254/latest/meta-data/local-ipv4`",
                                            "\n",
                                            "hostname=`/bin/hostname`",
                                            "\n",
                                            "echo \"$ipaddress  $hostname\" >> /etc/hosts",
                                            "\n",
                                            "sed -E 's/connectionTimeout=\"[[:digit:]]+\"/connectionTimeout=\"285000\"/g' /etc/tomcat/server.xml > /etc/tomcat/server.xml.improved",
                                            "\n",
                                            "mv -f /etc/tomcat/server.xml.improved /etc/tomcat/server.xml",
                                            "\n",
                                            "cat /etc/tomcat/server.xml",
                                            "\n",
                                            "export environment='", {
                                                "Ref": "GHXEnvironment"
                                            },

                                            "'\n",
                                            "export region='", {
                                                "Ref": "AWS::Region"
                                            },

                                            "'\n",
                                            "export ADDOPTS='", {
                                                "Ref": "AdditionalJavaOpts"
                                            },

                                            "'\n",
                                            "if [ -e /etc/tomcat/tomcat.blank ]; then",
                                            "\n",
                                            "   mv /etc/tomcat/tomcat.blank /etc/tomcat/tomcat.conf",
                                            "\n",
                                            "fi",
                                            "\n",
                                            "echo \"JAVA_OPTS='-Dghx.configsvc.db.url=", {
                                                "Ref": "configSVCurl"
                                            },

                                            " -Dghx.configsvc.db.pass=", {
                                                "Ref": "configSVCpass"
                                            },

                                            " $ADDOPTS ",
                                            "' \" >> /etc/tomcat/tomcat.conf\n",
                                            "\n",
                                            "echo 'export SHUTDOWN_WAIT=300' >> /etc/tomcat/tomcat.conf",
                                            "\n",
                                            "sudo service tomcat start",
                                            "\n"
                                        ]
                                    ]
                                },

                                "mode": "000777",
                                "owner": "root",
                                "group": "root"
                            }
                        },

                        "commands": {
                            "1-deploy-application": {
                                "command": "/var/tmp/deployScript/prep.sh"
                            }
                        }
                    }
                }
            },

            "Properties": {
                "IamInstanceProfile": {
                    "Fn::FindInMap": [
                        "Ec2Role", {
                            "Ref": "GHXEnvironment"
                        },

                        "Role"
                    ]
                },

                "ImageId": {
                    "Fn::FindInMap": ["AWSRegionArch2AMI", {
                        "Ref": "AWS::Region"
                    }, "AMIID"]
                },

                "SecurityGroups": [{
                    "Ref": "MercuryGruInstanceSecurityGroup"
                }],

                "InstanceType": {
                    "Ref": "InstanceType"
                },

                "KeyName": {
                    "Ref": "KeyName"
                },

                "InstanceMonitoring": {
                    "Ref": "InstanceMonitoring"
                },

                "UserData": {
                    "Fn::Base64": {
                        "Fn::Join": [
                            "", [
                                "#!/bin/bash",
                                "\n",
                                "yum update -y aws-cfn-bootstrap",
                                "\n",
                                "stackID='", {
                                    "Ref": "AWS::StackId"
                                },

                                "'\n",
                                "echo 'RUN CFN-INIT NEXT'",
                                "\n",
                                "/opt/aws/bin/cfn-init -v -c setupall -s $stackID -r MercuryGruLaunchConfig --region ", {
                                    "Ref": "AWS::Region"
                                },

                                "\n",
                                "echo 'CFN-INIT SHOULD HAVE JUST RUN'",
                                "\n"
                            ]
                        ]
                    }
                }
            }
        },

        "MercuryGruAutoScaling": {
            "Type": "AWS::AutoScaling::AutoScalingGroup",
            "Properties": {
                "VPCZoneIdentifier": [{
                        "Fn::GetAtt": [
                            "NetworkInfo",
                            "PrivateSubnet1"
                        ]
                    },

                    {
                        "Fn::GetAtt": [
                            "NetworkInfo",
                            "PrivateSubnet2"
                        ]
                    }
                ],
                "AvailabilityZones": [{
                        "Fn::GetAtt": [
                            "NetworkInfo",
                            "AZ1"
                        ]
                    },

                    {
                        "Fn::GetAtt": [
                            "NetworkInfo",
                            "AZ2"
                        ]
                    }
                ],
                "MinSize": {
                    "Ref": "ScalingMin"
                },
                "MaxSize": {
                    "Ref": "ScalingMax"
                },
                "LoadBalancerNames": [{
                    "Fn::GetAtt": [
                        "ELBInfo",
                        "LoadBalancerName"
                    ]
                }],
                "LaunchConfigurationName": {
                    "Ref": "MercuryGruLaunchConfig"
                },
                "HealthCheckType": "ELB",
                "HealthCheckGracePeriod": "780",
                "MetricsCollection": [{
                    "Granularity": "1Minute",
                    "Metrics": ["GroupInServiceInstances", "GroupTotalInstances", "GroupDesiredCapacity"]
                }],
                "Tags": [{
                        "Key": "uptimehours",
                        "Value": "24",
                        "PropagateAtLaunch": "true"
                    },

                    {
                        "Key": "service",
                        "Value": "corex-test-framework",
                        "PropagateAtLaunch": "true"
                    },

                    {
                        "Key": "owner",
                        "Value": "enhancedeliveryteam@ghx.com",
                        "PropagateAtLaunch": "true"
                    },

                    {
                        "Key": "Name",
                        "Value": {
                            "Ref": "TagName"
                        },

                        "PropagateAtLaunch": "true"
                    },
                    {
                        "Key": "newrelic",
                        "Value": {
                            "Ref": "NewRelicEnabled"
                        },
                        "PropagateAtLaunch": "true"
                    }
                ]
            }
        }
    }
}

tfw-minion-cf-template.json

{
    "AWSTemplateFormatVersion": "2010-09-09",
    "Description": "Test Framework Minion stack",
    "Parameters": {
        "VPCStackName": {
            "Description": "Name of the VPC CloudFormaiton Stack",
            "Type": "String",
            "Default": "LegacyVpcCfOutputs"
        },
        "InstanceType": {
            "Description": "WebServer EC2 instance type",
            "Type": "String",
            "Default": "m5a.large",
            "AllowedValues":[
                "m3.medium",
                "m3.large",
                "m4.large",
                "m4.xlarge",
                "m4.2xlarge",
                "m5a.large",
                "m5a.xlarge",
                "t3a.medium",
                "t3a.small",
                "t2.medium",
                "t2.small"
            ],
            "ConstraintDescription": "must be a valid EC2 instance type."
        },
        "KeyName": {
            "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
            "Type": "AWS::EC2::KeyPair::KeyName",
            "ConstraintDescription": "must be the name of an existing EC2 KeyPair."
        },
        "configSVCurl": {
            "Description": "The URL of the configsvc to use",
            "Type": "String",
            "ConstraintDescription": "must be the name of an existing configsvc mongo database."
        },
        "testFrameWorkURL": {
            "Description": "The URL of the testframework to use",
            "Type": "String",
            "ConstraintDescription": "must be the name of an existing testframework mongo database."
        },
        "configSVCpass": {
            "Description": "The encrypted password of the configsvc to use",
            "Type": "String",
            "ConstraintDescription": "must be the password of an existing configsvc mongo database."
        },
        "AdditionalJavaOpts": {
            "Description": "Additional JAVA_OPTS as system properties, -Dname=value",
            "Type": "String",
            "Default": "",
            "ConstraintDescription": "Set as system property, -D"
        },
        "ScalingMax": {
            "Description": "Max Number to scale EC2 instances to",
            "Type": "Number",
            "Default": "1",
            "MinValue": "1",
            "MaxValue": "350"
        },
        "ScalingMin": {
            "Description": "Min Number to scale EC2 instances to",
            "Type": "Number",
            "Default": "1",
            "MinValue": "1",
            "MaxValue": "200"
        },
        "MongoConfigDB":{
            "Description":"Connection String to the Mongo Shard Config database. Only used if MongosEnabled=true",
            "Type":"String",
            "Default":"UNKNOWN",
            "ConstraintDescription":"must be a valid mongodb url"
        },
        "MongosEnabled":{
            "Description":"Select true to enable Mongos to allow AppServer to connect to a shard",
            "Type":"String",
            "Default":"false",
            "AllowedValues":[
                "true",
                "false"
            ],
            "ConstraintDescription":"must be true or false"
        },
        "Service" : {
            "Description" : "Value to apply to the service tag for instances",
            "Type": "String",
            "Default": "test-framework-minion"
        },
        "InstanceMonitoring" : {
            "Description" : "Set to false for Basic monitoring and true for Detailed monitoring",
            "Type": "String",
            "Default": "false"
        },
        "Application":{
            "Description":"Application   To Map To Application ",
            "Type":"String",
            "ConstraintDescription":"Must Be Valid Application Label"
        },
        "War": {
            "Description": "The War Name",
            "Type": "String",
            "ConstraintDescription": "Name of the war to deployed"
        },
        "DeploymentBucket" : {
            "Description" : "Set to true to create SNS notifications for instance_terminations of ASGs",
            "Type" : "String",
            "Default" : "ghx-deployment"
        },
        "GHXEnvironment": {
            "Description": "GHX environment",
            "Type": "String",
            "AllowedValues": [
                "CUSTOM",
                "DEV",
                "STG",
                "DEVINT",
                "CoreXBINT",
                "PRD",
                "LOAD"
            ]

        },
        "TagName": {
            "Description": "Name of EC2 instances launched from AutoScalingGroup",
            "Type": "String"
        },
        "ApplicationVersion": {
            "Description": "Application Version Number To Map To Application Key In S3",
            "Type": "String",
            "ConstraintDescription": "Must Be Valid Application Version Label"
        },
        "BranchName": {
            "Description": "Bamboo Branch Name",
            "Type": "String",
            "ConstraintDescription": "Must be valid branch name"
        },
        "BuildNumber": {
            "Description": "Bamboo build number",
            "Type": "String",
            "ConstraintDescription": "must be a valid bamboo build number"
        },
        "TestResultId":{
            "Description":"The Id of scenario in case if stack creates by testframework",
            "Type":"String",
            "Default":"NA"
        },
        "Product":{
            "Description":"Product name. Used to determine asset locations in deployment bucket",
            "Type":"String",
            "Default": "corex"
        },
        "NewrelicApiUrl":{
            "Description":"URL of Newrelic API",
            "Type":"String",
            "Default":"NA"
        },
        "NewrelicApiInsightUrl":{
            "Description":"Url of Newrelic Api Insight",
            "Type":"String",
            "Default":"NA"
        },
        "NewrelicApiKey":{
            "Description":"API Key of Newrelic",
            "Type":"String",
            "Default":"NA"
        },
        "NewrelicQueryKey":{
            "Description":"Query Key of Newrelic",
            "Type":"String",
            "Default":"NA"
        },
    "CloudMongoApiUrl":{
        "Description":"Url of Cloud Mongo Api",
            "Type":"String",
            "Default":"NA"
        },
    "CloudMongoMMSpublickey":{
        "NoEcho" : "true",
         "Description":"Cloud Mongo MMS public key",
            "Type":"String",
            "Default":"NA"
    },
    "CloudMongoMMSprivatekey":{
        "NoEcho" : "true",
         "Description":"Cloud Mongo MMS private key",
            "Type":"String",
            "Default":"NA"
    },
    "ebExecutionURL":{
        "NoEcho" : "true",
         "Description":"eb-execution DB mongo url",
            "Type":"String",
            "Default":"NA"
    },

    "eventbusUrl":{
        "NoEcho" : "true",
         "Description":"Url of mongo DB Event Bus",
        "Type":"String",
            "Default":"NA"
    }
    },
    "Conditions" : {
        "IsNotATestStack" : {"Fn::Equals" : [{"Ref":"TestResultId"}, "NA"]}
    },
    "Mappings": {
        "AWSRegionArch2AMI": {
            "us-east-1": {
                "AMIID": "TOKENMINIONAMIUSEAST1"
            },
            "us-west-2": {
                "AMIID": "TOKENMINIONAMIUSWEST2"
            }

        },
        "Ec2Role": {
            "CUSTOM": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },
            "DEV": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },
            "STG": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },
            "DEVINT": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },
            "CoreXBINT": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            },
            "PRD": {
                "Role": "arn:aws:iam::326626545197:instance-profile/DevOpsRole-test-framework"
            },
            "LOAD": {
                "Role": "arn:aws:iam::050190852521:instance-profile/DevOpsRole-test-framework"
            }
        }
    },
    "Resources": {
        "NetworkInfo": {
            "Type": "Custom::NetworkInfo",
            "Properties": {
                "ServiceToken": {
                    "Fn::Join": ["", [
                        "arn:aws:lambda:", {
                            "Ref": "AWS::Region"
                        }, ":", {
                            "Ref": "AWS::AccountId"
                        },
                        ":function:DescribeCFStackOutputs"
                    ]]
                },
                "StackName": {
                    "Ref": "VPCStackName"
                }
            }
        },
        "MercuryMinionInstanceSecurityGroup": {
            "Type": "AWS::EC2::SecurityGroup",
            "Properties": {
                "GroupDescription": "Security Group for Minion Stack EC2 Instances",
                "SecurityGroupEgress": [{
                        "CidrIp": "0.0.0.0/0",
                        "FromPort": "0",
                        "IpProtocol": "tcp",
                        "ToPort": "65535"
                    }, {
                        "CidrIp": "0.0.0.0/0",
                        "FromPort": "0",
                        "IpProtocol": "udp",
                        "ToPort": "65535"
                    }, {
                        "CidrIp": "0.0.0.0/0",
                        "FromPort": "0",
                        "IpProtocol": "-1",
                        "ToPort": "65535"
                    }

                ],
                "SecurityGroupIngress": [{
                        "CidrIp": "0.0.0.0/0",
                        "FromPort": "0",
                        "IpProtocol": "-1",
                        "ToPort": "65535"
                    }

                ],
                "VpcId": {
                    "Fn::GetAtt": [
                        "NetworkInfo",
                        "VpcId"
                    ]
                }

            }

        },
        "MercuryMinionLaunchConfig": {
            "Type": "AWS::AutoScaling::LaunchConfiguration",
            "Metadata": {
                "AWS::CloudFormation::Init": {
                    "configSets": {
                        "setupall": [
                            "paramfile",
                            "logstash",
                            "cpumetriccollection",
                            "tpmmetriccollection",
                            "hikarimetriccollection",
                            "heapmetriccollection",
                            "dynamometriccollection",
                            "eventbusmongometriccollection",
                            "cloudmongocollection",
                            "eventbustroubleshooting",
                            "mongos",
                            "deploy"
                        ]

                    },
                    "paramfile": {
                        "files": {
                            "/opt/conf/cf-params.txt": {
                                "content": {
                                    "Fn::Join": ["", [
                                        "accountid:", { 
                                            "Ref": "AWS::AccountId"
                                        }, "\n",
                                        "AdditionalJavaOpts:", {
                                            "Ref": "AdditionalJavaOpts"
                                        }, "\n",
                                        "ApplicationVersion:", {
                                            "Ref": "ApplicationVersion"
                                        }, "\n",
                                        "BranchName:", {
                                            "Ref": "BranchName"
                                        }, "\n",
                                        "BuildNumber:", {
                                            "Ref": "BuildNumber"
                                        }, "\n",
                                        "CFStackName:", {
                                            "Ref": "AWS::StackName"
                                        }, "\n",
                                        "service:",{ 
                                            "Ref" : "Service" 
                                        },"\n",
                                        "MongoConfigDB:",{
                                            "Ref": "MongoConfigDB"
                                        }, "\n",
                                        "MongosEnabled:",{
                                            "Ref": "MongosEnabled" 
                                        }, "\n",
                                        "Application:",{ 
                                            "Ref" : "Application" 
                                        },"\n",
                                        "GHXEnvironment:", {
                                            "Ref": "GHXEnvironment"
                                        }, "\n",
                                        "InstanceType:", {
                                            "Ref": "InstanceType"
                                        }, "\n",
                                        "KeyName:", {
                                            "Ref": "KeyName"
                                        }, "\n",
                                        "region:", {
                                            "Ref": "AWS::Region"
                                        }, "\n",
                                        "ScalingMax:", {
                                            "Ref": "ScalingMax"
                                        }, "\n",
                                        "ScalingMin:", {
                                            "Ref": "ScalingMin"
                                        }, "\n",
                                        "War:", {
                                            "Ref": "War"
                                        }, "\n",
                                        "DeploymentBucket:", {
                                            "Ref": "DeploymentBucket"
                                        }, "\n",
                                        "Product:", { 
                                            "Ref": "Product" 
                                        }, "\n",
                                        "TestResultId:", { 
                                            "Ref": "TestResultId" 
                                        }, "\n",
                                        "NewrelicApiUrl:", { 
                                            "Ref": "NewrelicApiUrl" 
                                        }, "\n",
                                        "NewrelicApiInsightUrl:", { 
                                            "Ref": "NewrelicApiInsightUrl" 
                                        }, "\n",
                                        "NewrelicApiKey:", { 
                                            "Ref": "NewrelicApiKey" 
                                        }, "\n",
                                        "NewrelicQueryKey:", { 
                                            "Ref": "NewrelicQueryKey" 
                                        }, "\n",
                                        "CloudMongoApiUrl:", { 
                                            "Ref": "CloudMongoApiUrl" 
                                        }, "\n",
                                        "CloudMongoMMSpublickey:", {
                                            "Ref": "CloudMongoMMSpublickey"
                                        }, "\n",
                                        "CloudMongoMMSprivatekey:", {
                                            "Ref": "CloudMongoMMSprivatekey"
                                        }, "\n",
                                        "ebExecutionURL:", {
                                            "Ref": "ebExecutionURL"
                                        }, "\n",
                                        "testFrameWorkURL:", {
                                            "Ref":"testFrameWorkURL"                                    
                                        }, "\n",    
                                        "eventbusUrl:", {
                                            "Ref":"eventbusUrl"
                                        }, "\n"
                                    ]]
                                },
                                "mode": "000644",
                                "owner": "root",
                                "group": "root"
                            }
                    },
                    
                        "commands": {
                            "01-createdirs": {
                                "command": "/bin/mkdir /opt/scripts; /bin/mkdir -p /opt/conf/; /bin/mkdir -p /var/log/cpumetric/;"
                            }

                        }
                    },
                    "logstash": {
                        "commands": {
                            "01-get-logstash-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-config-logstash-jmx.py /opt/scripts/cf-config-logstash-jmx.py",
                                            " ; chmod 744 /opt/scripts/cf-config-logstash-jmx.py"
                                        ]
                                    ]
                                }
                            },
                            "05-config-logstash": {
                                "command": "/opt/scripts/cf-config-logstash-jmx.py"
                            },
                            "11-get-start-ls": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-ls.sh /opt/scripts/start-ls.sh",
                                            " ; chmod 744 /opt/scripts/start-ls.sh"
                                        ]
                                    ]
                                }
                            },
                            "15-start-logstash": {
                                "command": "/opt/scripts/start-ls.sh"
                            }
                        }
                    },
                    "cpumetriccollection": {
                        "commands": {   
             "01-get-testfwmetrics-cpu-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-cpumetriccollection.py /opt/scripts/cf-start-cpumetriccollection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-cpumetriccollection.py"
                                        ]
                                    ]
                                }
                            },
                "02-get-testfwmetrics-newrelic-cpu-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                    [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-newrelic-cpumetriccollection.py /opt/scripts/cf-start-newrelic-cpumetriccollection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-newrelic-cpumetriccollection.py"
                                        ]
                                    ]
                                }
                            },
                            "11-get-start-cpu": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-cpu.sh /opt/scripts/start-cpu.sh",
                                            " ; chmod 744 /opt/scripts/start-cpu.sh"
                                        ]
                                    ]
                                }
                            },
                            "12-get-crontab": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/crontab /tmp/crontab",
                                            " ; cat /tmp/crontab >> /etc/crontab"
                                        ]
                                    ]
                                }
                            },
                            "15-start-cpu": {
                                "command": "/opt/scripts/start-cpu.sh"
                            }
                        }
                    },
                    "tpmmetriccollection": {
                        "commands": {   
                            "01-get-testfwmetrics-newrelic-tpm-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                    [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-newrelic-tpmmetriccollection.py /opt/scripts/cf-start-newrelic-tpmmetriccollection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-newrelic-tpmmetriccollection.py"
                                        ]
                                    ]
                                }
                            },
                            "02-get-start-tpm": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-tpm.sh /opt/scripts/start-tpm.sh",
                                            " ; chmod 744 /opt/scripts/start-tpm.sh"
                                        ]
                                    ]
                                }
                            },
                            "03-start-tpm": {
                                "command": "/opt/scripts/start-tpm.sh"
                            }
                        }
                    },
                    "hikarimetriccollection": {
                        "commands": {   
                            "01-get-testfwmetrics-newrelic-hikari-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                    [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-newrelic-hikarimetriccollection.py /opt/scripts/cf-start-newrelic-hikarimetriccollection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-newrelic-hikarimetriccollection.py"
                                        ]
                                    ]
                                }
                            },
                            "02-get-start-hikari": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-hikari.sh /opt/scripts/start-hikari.sh",
                                            " ; chmod 744 /opt/scripts/start-hikari.sh"
                                        ]
                                    ]
                                }
                            },
                            "03-start-hikari": {
                                "command": "/opt/scripts/start-hikari.sh"
                            }
                        }
                    },
                    "heapmetriccollection": {
                        "commands": {   
                    "01-get-testfwmetrics-newrelic-heap-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                    [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-newrelic-heapmetriccollection.py /opt/scripts/cf-start-newrelic-heapmetriccollection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-newrelic-heapmetriccollection.py"
                                        ]
                                    ]
                                }
                            },
                            "02-get-start-heap": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-heap.sh /opt/scripts/start-heap.sh",
                                            " ; chmod 744 /opt/scripts/start-heap.sh"
                                        ]
                                    ]
                                }
                            },
                            "03-start-heap": {
                                "command": "/opt/scripts/start-heap.sh"
                        }
                      }
                       },
                       "cloudmongocollection":{
                        "commands":{
                               "01-get-testfwmetrics-cloudmongo-script":{
                                "command":{
                                    "Fn::Join":[
                                          "",
                                          [
                                          "aws s3 cp s3://ghx-deployment/corex/",
                                           {"Ref":"ApplicationVersion"},"/",
                                           {"Ref":"BranchName"},"/",
                                           {"Ref":"BuildNumber"},
                                           "/cf-deploy/scripts/cf-start-cloudmongo-eventbuscollection.py /opt/scripts/cf-start-cloudmongo-eventbuscollection.py",
                                           " ; chmod 744 /opt/scripts/cf-start-cloudmongo-eventbuscollection.py"
                                          ]
                                     ]
                                 }
                             },
                             "02-get-start-cloudmongo":{
                             "command":{
                                   "Fn::Join":[
                                       "",
                                       [
                                           "aws s3 cp s3://ghx-deployment/corex/",
                                         {"Ref":"ApplicationVersion"},"/",
                                         {"Ref":"BranchName"},"/",
                                         {"Ref":"BuildNumber"},
                                          "/cf-deploy/scripts/start-cloudmongo.sh /opt/scripts/start-cloudmongo.sh",
                                          " ; chmod 744 /opt/scripts/start-cloudmongo.sh"
                                         ]
                                   ]
                                 }
                              },
                               "03-start-cloudmongo":{
                              "command":"/opt/scripts/start-cloudmongo.sh"
                               }
                         }
                       },
            "dynamometriccollection": {
                         "commands": {   
             "01-get-testfwmetrics-dynamodb-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-aws-dynamometric-collection.py /opt/scripts/cf-start-aws-dynamometric-collection.py",
                                            " ; chmod 744 /opt/scripts/cf-start-aws-dynamometric-collection.py"
                                        ]
                                    ]
                                }
                           },
               "02-get-start-dynamodb": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-dynamo.sh /opt/scripts/start-dynamo.sh",
                                            " ; chmod 744 /opt/scripts/start-dynamo.sh"
                                        ]
                                    ]
                                }
                            },
                "03-start-dynamodb": {
                                "command": "/opt/scripts/start-dynamo.sh"
                                }
                }
                        },
               "eventbustroubleshooting": {
                             "commands": {   
                 "01-get-eventbus-troubleshooting-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-start-eventbus-troubleshootingmetrics.py /opt/scripts/cf-start-eventbus-troubleshootingmetrics.py",
                                            " ; chmod 744 /opt/scripts/cf-start-eventbus-troubleshootingmetrics.py"
                                        ]
                                    ]
                                }
                           },
               "02-get-start-eventbus-troubleshooting": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://ghx-deployment/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/start-eventbustroubleshooting.sh /opt/scripts/start-eventbustroubleshooting.sh",
                                            " ; chmod 744 /opt/scripts/start-eventbustroubleshooting.sh"
                                        ]
                                    ]
                                }
                            },
                "03-start-eventbus-troubleshooting": {
                                "command": "/opt/scripts/start-eventbustroubleshooting.sh"
                                }
                }
                        },
             "eventbusmongometriccollection": {
                             "commands": {   
                 "01-get-testfwmetrics-eventbus-script": {
                                  "command": {
                                        "Fn::Join": [
                                          "",
                                          [
                                             "aws s3 cp s3://ghx-deployment/corex/",
                                             { "Ref": "ApplicationVersion" },"/",
                                             { "Ref": "BranchName" },"/",
                                             { "Ref": "BuildNumber" },
                                             "/cf-deploy/scripts/cf-start-mongo-eventbusmetric-collection.py /opt/scripts/cf-start-mongo-eventbusmetric-collection.py",
                                             " ; chmod 744 /opt/scripts/cf-start-mongo-eventbusmetric-collection.py"
                                          ]
                                        ]
                                   }
                              },
                  "02-get-start-eventbusmongo": {
                                   "command": {
                                        "Fn::Join": [
                                           "",
                                           [
                                                "aws s3 cp s3://ghx-deployment/corex/",
                                                { "Ref": "ApplicationVersion" },"/",
                                                { "Ref": "BranchName" },"/",
                                                { "Ref": "BuildNumber" },
                                                "/cf-deploy/scripts/start-eventbus.sh /opt/scripts/start-eventbus.sh",
                                                " ; chmod 744 /opt/scripts/start-eventbus.sh"
                                           ]
                                         ]
                                    }
                               },
                   "03-start-eventbus": {
                                    "command": "/opt/scripts/start-eventbus.sh"
                    }
                   }
                    },
                    "mongos": {
                        "commands": {
                            "01-get-mongos-script": {
                                "command": {
                                    "Fn::Join": [
                                        "",
                                        [
                                            "aws s3 cp s3://",
                                            { "Ref": "DeploymentBucket" },
                                            "/corex/",
                                            { "Ref": "ApplicationVersion" },"/",
                                            { "Ref": "BranchName" },"/",
                                            { "Ref": "BuildNumber" },
                                            "/cf-deploy/scripts/cf-config-mongos.py /opt/scripts/cf-config-mongos.py",
                                            " ; chmod 744 /opt/scripts/cf-config-mongos.py"
                                        ]
                                    ]
                                }
                            },
                            "05-config-mongos": {
                                "command": "/opt/scripts/cf-config-mongos.py"
                            }
                        }
                    },
                    "deploy": {
                        "files": {
                            "/etc/cfn/cfn-hup.conf": {
                                "content": {
                                    "Fn::Join": [
                                        "", [
                                            "[main]\n",
                                            "stack=", {
                                                "Ref": "AWS::StackId"
                                            },
                                            "\n",
                                            "region=", {
                                                "Ref": "AWS::Region"
                                            },
                                            "\n",
                                            "interval=1\n"
                                        ]

                                    ]

                                }

                            },
                            "/opt/scripts/MonitoringDependentAppStatus.sh": {
                                "content": {
                                    "Fn::Join": [
                                          "", [
                                          "#!/usr/bin/env bash",
                                          "\n",
                                         "logstash=0;\n",
                                           "host=$(hostname -I | awk '{print $1}')",
                                           "\n",
                                          "logstash=$(ps -ef |grep logstash |wc -l)",
                                          "\n",
                                        "while [ $logstash != 2 ]",
                                        "\n",
                                        "do",
                                        "\n",
                                        "logstash=$(ps -ef |grep logstash |wc -l)",
                                        "\n",
                                        "sleep 2",
                                        "\n",
                                        "done",
                                        "\n",
                                          "if [ $logstash == 2 ]",
                                          "\n",
                                          "then",
                                          "\n",
                                          "dbconnection=`ps -ef |grep tomcat | grep -Po 'test-framework.db.url=\\K(.*)' | cut -d' ' -f1`",
                                          "\n",
                                         "CMD=\"printjson(db.MonitoringDependentAppStatus.remove({'scenarioType':'CUSTOM','minionIP':'$host','applicationName':'logstash'}),db.MonitoringDependentAppStatus.insert({'scenarioType':'CUSTOM','minionIP':'$host','applicationName':'logstash','created':{'by':'system','on':ISODate()}}))\"",
                                          "\n",
                                          " /usr/bin/mongo $dbconnection --eval  $CMD",
                                          "\n",
                                          "fi",
                                          "\n"
                                              ]
                                    ]
                                },
                                  "mode": "000777",
                                  "owner": "root",
                                  "group": "root"
                                },
                            "/var/tmp/deployScript/prep.sh": {
                                "content": {
                                    "Fn::Join": [
                                        "", [
                                            "#!/bin/bash",
                                            "\n",
                                            "date > /home/ec2-user/starttime\n",
                                            "ipaddress=`curl http://169.254.169.254/latest/meta-data/local-ipv4`",
                                            "\n",
                                            "hostname=`/bin/hostname`",
                                            "\n",
                                            "echo \"$ipaddress  $hostname\" >> /etc/hosts",
                                            "\n",
                                            "sed -E 's/connectionTimeout=\"[[:digit:]]+\"/connectionTimeout=\"285000\"/g' /etc/tomcat/server.xml > /etc/tomcat/server.xml.improved",
                                            "\n",
                                            "mv -f /etc/tomcat/server.xml.improved /etc/tomcat/server.xml",
                                            "\n",
                                            "cat /etc/tomcat/server.xml",
                                            "\n",
                                            "export environment='", {
                                                "Ref": "GHXEnvironment"
                                            },
                                            "'\n",
                                            "export region='", {
                                                "Ref": "AWS::Region"
                                            },
                                            "'\n",
                                            "export ADDOPTS='", {
                                                "Ref": "AdditionalJavaOpts"
                                            },
                                            "'\n",
                                            "if [ -e /etc/tomcat/tomcat.blank ]; then",
                                            "\n",
                                            "   mv /etc/tomcat/tomcat.blank /etc/tomcat/tomcat.conf",
                                            "\n",
                                            "fi",
                                            "\n",
                                            "echo \"JAVA_OPTS='-Dlogger.level.egx=DEBUG -Dghx.configsvc.db.url=", {
                                                "Ref": "configSVCurl"
                                            },
                                            " -Dghx.test-framework.db.url=", {
                                                "Ref": "testFrameWorkURL"
                                            },
                                            " -Dghx.configsvc.db.pass=", {
                                                "Ref": "configSVCpass"
                                            },
                                            " $ADDOPTS ",
                                            "' \" >> /etc/tomcat/tomcat.conf\n",
                                            "echo \"JMETER_HOME=/install/apache-jmeter-3.2 \" >> /etc/tomcat/tomcat.conf\n",
                                            "\n",
                                            "echo 'export SHUTDOWN_WAIT=300' >> /etc/tomcat/tomcat.conf",
                                            "\n",
                                            "sudo service tomcat start",
                                            "\n",
                                            "sleep 30s",
                                            "\n",
                                            "sh /opt/scripts/MonitoringDependentAppStatus.sh",
                                            "\n"
                                        ]

                                    ]

                                },
                                "mode": "000777",
                                "owner": "root",
                                "group": "root"
                            }
                        },
                        "commands": {
                            "1-deploy-application": {
                                "command": "/var/tmp/deployScript/prep.sh"
                            }

                        }

                    }

                }

            },
            "Properties": {
                "IamInstanceProfile": {
                    "Fn::FindInMap": [
                        "Ec2Role", {
                            "Ref": "GHXEnvironment"
                        },
                        "Role"
                    ]

                },
                "ImageId": {
                    "Fn::FindInMap": [ "AWSRegionArch2AMI", { "Ref": "AWS::Region" }, "AMIID" ]
                },
                "SecurityGroups": [{
                        "Ref": "MercuryMinionInstanceSecurityGroup"
                    }
                ],
                "InstanceType": {
                    "Ref": "InstanceType"
                },
                "KeyName": {
                    "Ref": "KeyName"
                },
                "InstanceMonitoring": {
                    "Ref": "InstanceMonitoring"
                },
                "UserData": {
                    "Fn::Base64": {
                        "Fn::Join": [
                            "", [
                                "#!/bin/bash",
                                "\n",
                                "yum update -y aws-cfn-bootstrap",
                                "\n",
                                "stackID='", {
                                    "Ref": "AWS::StackId"
                                },
                                "'\n",
                                "echo 'RUN CFN-INIT NEXT'",
                                "\n",
                                "/opt/aws/bin/cfn-init -v -c setupall -s $stackID -r MercuryMinionLaunchConfig --region ", {
                                    "Ref": "AWS::Region"
                                },
                                "\n",
                                "echo 'CFN-INIT SHOULD HAVE JUST RUN'",
                                "\n"
                            ]

                        ]

                    }

                }

            }

        },
        "MercuryMinionAutoScaling": {
            "Type": "AWS::AutoScaling::AutoScalingGroup",
            "Properties": {
                "AvailabilityZones": [{
                    "Fn::GetAtt": [
                        "NetworkInfo",
                        "AZ1"
                    ]
                }, {
                    "Fn::GetAtt": [
                        "NetworkInfo",
                        "AZ2"
                    ]
                }],
                "MinSize": {
                    "Ref": "ScalingMin"
                },
                "MaxSize": {
                    "Ref": "ScalingMax"
                },
                "LaunchConfigurationName": {
                    "Ref": "MercuryMinionLaunchConfig"
                },
                "VPCZoneIdentifier": [{
                    "Fn::GetAtt": [
                        "NetworkInfo",
                        "PrivateSubnet1"
                    ]
                }, {
                    "Fn::GetAtt": [
                        "NetworkInfo",
                        "PrivateSubnet2"
                    ]
                }],
                "Tags": [{
                        "Key": "uptimehours",
                        "Value": "24",
                        "PropagateAtLaunch": "true"
                    }, {
                        "Key": "service",
                        "Value": "corex-test-framework",
                        "PropagateAtLaunch": "true"
                    }, {
                        "Key": "owner",
                        "Value": "enhancedeliveryteam@ghx.com",
                        "PropagateAtLaunch": "true"
                    }, {
                        "Key": "Name",
                        "Value": {
                            "Ref": "TagName"
                        },
                        "PropagateAtLaunch": "true"
                    }

                ]

            }

        }

    }


}

tfw-nlb-regapi.json:

{
   "AWSTemplateFormatVersion":"2010-09-09",
   "Description":"Deploy NLB for Corex RegAPI Component For INT and PROD",
   "Parameters":{
      "VPCStackName":{
         "Description":"Name of the VPC CloudFormation Stack",
         "Type":"String",
         "Default":"LegacyVpcCfOutputs"
      },
      "Service":{
         "Description":"Service tag value to apply to the ELB, must be a valid, approved service value",
         "Type":"String"
      },
      "GHXEnvironment":{
         "Description":"GHX Environment Name",
         "Type":"String"
      },
      "TargetGroupName":{
         "Description":"RegapiTargetGroup Name",
         "Type":"String"
      }
   },
   "Resources":{
      "NetworkInfo":{
         "Type":"Custom::NetworkInfo",
         "Properties":{
            "ServiceToken":{
               "Fn::Join":[
                  "",
                  [
                     "arn:aws:lambda:",
                     {
                        "Ref":"AWS::Region"
                     },
                     ":",
                     {
                        "Ref":"AWS::AccountId"
                     },
                     ":function:DescribeCFStackOutputs"
                  ]
               ]
            },
            "StackName":{
               "Ref":"VPCStackName"
            }
         }
      },
      "RegapiNLB":{
         "Type":"AWS::ElasticLoadBalancingV2::LoadBalancer",
         "Properties":{
            "LoadBalancerAttributes":[
               {
                  "Key":"deletion_protection.enabled",
                  "Value":"false"
               },
               {
                  "Key":"load_balancing.cross_zone.enabled",
                  "Value":"true"
               }
            ],
            "Name":{
               "Ref":"AWS::StackName"
            },
            "Scheme":"internal",
            "Subnets":[
               {
                  "Fn::GetAtt":[
                     "NetworkInfo",
                     "LegacyPublicSubnet1"
                  ]
               },
               {
                  "Fn::GetAtt":[
                     "NetworkInfo",
                     "LegacyPublicSubnet2"
                  ]
               },
               {
                  "Fn::GetAtt":[
                     "NetworkInfo",
                     "LegacyPublicSubnet3"
                  ]
               }
            ],
            "Type":"network",
            "Tags":[
               {
                  "Key":"uptimehours",
                  "Value":"24"
               },
               {
                  "Key":"service",
                  "Value":{
                     "Ref":"Service"
                  }
               },
               {
                  "Key":"owner",
                  "Value":"gss-changecontrol@ghx.com"
               },
               {
                  "Key":"GHXEnvironment",
                  "Value":{
                     "Ref":"GHXEnvironment"
                  }
               }
            ]
         }
      },
      "RegapiTargetGroup":{
         "Type":"AWS::ElasticLoadBalancingV2::TargetGroup",
         "Properties":{
            "Name":{
               "Fn::Join":[
                  "",
                  [
                     "TFWRG-",
                     {
                        "Ref":"TargetGroupName"
                     }
                  ]
               ]
            },
            "Port":8080,
            "Protocol":"TCP",
            "HealthCheckIntervalSeconds":10,
            "HealthyThresholdCount":2,
            "UnhealthyThresholdCount":2,
            "TargetType":"instance",
            "TargetGroupAttributes":[
               {
                  "Key":"deregistration_delay.timeout_seconds",
                  "Value":300
               }
            ],
            "VpcId":{
               "Fn::GetAtt":[
                  "NetworkInfo",
                  "VpcId"
               ]
            },
            "Tags":[
               {
                  "Key":"service",
                  "Value":"regapi"
               }
            ]
         }
      },
      "RegapiHTTPListener":{
         "Type":"AWS::ElasticLoadBalancingV2::Listener",
         "Properties":{
            "DefaultActions":[
               {
                  "TargetGroupArn":{
                     "Ref":"RegapiTargetGroup"
                  },
                  "Type":"forward"
               }
            ],
            "LoadBalancerArn":{
               "Ref":"RegapiNLB"
            },
            "Port":80,
            "Protocol":"TCP"
         }
      },
      "RegapiHTTPSListener":{
         "Type":"AWS::ElasticLoadBalancingV2::Listener",
         "Properties":{
            "DefaultActions":[
               {
                  "TargetGroupArn":{
                     "Ref":"RegapiTargetGroup"
                  },
                  "Type":"forward"
               }
            ],
            "LoadBalancerArn":{
               "Ref":"RegapiNLB"
            },
            "Port":443,
            "Protocol":"TCP"
         }
      }
   },
   "Outputs":{
      "NlbDnsName":{
         "Description":"The DNS Name of the Network Load Balancer",
         "Value":{
            "Fn::GetAtt":[
               "RegapiNLB",
               "DNSName"
            ]
         }
      },
      "NlbName":{
         "Description":"Name of the Network LoadBalancer",
         "Value":{
            "Fn::GetAtt":[
               "RegapiNLB",
               "LoadBalancerName"
            ]
         }
      },
      "NlbARN":{
         "Description":"Name of the Network LoadBalancer",
         "Value":{
            "Ref":"RegapiNLB"
         }
      },
      "RegapiTargetGroupARN":{
         "Description":"The arn of the SFTP Target Group",
         "Value":{
            "Ref":"RegapiTargetGroup"
         }
      }
   }
}

ES template:
Corex-aactivityflowtemplate.json

{
"template": "corex_activityflow*",
"version": 109700,
"order": 0,
"settings": {
"index": {
"analysis": {
"normalizer": {
"lowercase_normalizer": {
"filter": [
"lowercase"
],
"type": "custom"
}
},
"analyzer": {
"custom_keyword": {
"filter": "lowercase",
"type": "custom",
"tokenizer": "keyword"
},
"url_analyzer": {
"type": "custom",
"tokenizer": "uax_url_email"
}
}
},
"warmer": {
"enabled": "false"
},
"query": {
"default_field": "DocType"
},
"max_result_window": "1000000",
"refresh_interval": "5s",
"number_of_shards": "5",
"number_of_replicas": "0"
}
},
"mappings": {
"metadata": {
"_source": {
"enabled": true
},
"properties": {
"Sender": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"Receiver": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"DSID": {
"analyzer": "custom_keyword",
"type": "string"
},
"AlertAmount": {
"null_value": "0.0",
"type": "double"
},
"FlowStartTime": {
"type": "date",
"fields": {
"untouched": {
"index": "not_analyzed",
"type": "string"
}
}
},
"InboundISASender": {
"analyzer": "custom_keyword",
"type": "string"
},
"ISAReceiver": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundISAReceiver": {
"analyzer": "custom_keyword",
"type": "string"
},
"DocType": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"DocVersion": {
"analyzer": "custom_keyword",
"type": "string"
},
"TrackingFlowId": {
"type": "keyword"
},
"OrgId": {
"type": "long"
},
"MOID": {
"type": "string"
},
"GSCount": {
"type": "integer"
},
"ShipTo": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"AdapterId": {
"type": "long"
},
"InboundGSReceiver": {
"analyzer": "custom_keyword",
"type": "string"
},
"OriginalTimestamp": {
"type": "date"
},
"CurrentActivityStatus": {
"analyzer": "custom_keyword",
"type": "string"
},
"OutboundChannel": {
"type": "string"
},
"PoolId": {
"type": "string"
},
"InboundMessageURL": {
"analyzer": "url_analyzer",
"type": "string"
},
"OutboundTimestamp": {
"type": "date"
},
"ISACount": {
"type": "integer"
},
"InboundChannel": {
"type": "string"
},
"ControlNumber": {
"analyzer": "custom_keyword",
"type": "string"
},
"GroupControlNumber": {
"analyzer": "custom_keyword",
"type": "string"
},
"GSControlNumber": {
"analyzer": "custom_keyword",
"type": "string"
},
"BillTo": {
"analyzer": "custom_keyword",
"type": "string"
},
"TrafficMode": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"InboundDocVersion": {
"analyzer": "custom_keyword",
"type": "string"
},
"ReceiverOrgId": {
"type": "long"
},
"InboundFilename": {
"analyzer": "custom_keyword",
"type": "string"
},
"SenderOrgId": {
"type": "long"
},
"SenderOrgCountryCode": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"From": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"To": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"InboundDocSubType": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"FlowEndTime": {
"type": "date",
"fields": {
"untouched": {
"index": "not_analyzed",
"type": "string"
}
}
},
"TotalDuration": {
"type": "long"
},
"DocSubType": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"PONumber": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"BusinessProcessIdList": {
"type": "string"
},
"InboundDocType": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"GSReceiver": {
"analyzer": "custom_keyword",
"type": "string"
},
"FlowId": {
"type": "keyword"
},
"ASNNumber": {
"analyzer": "custom_keyword",
"type": "string"
},
"BusinessProcessId": {
"type": "string"
},
"ISASender": {
"analyzer": "custom_keyword",
"type": "string"
},
"InvoiceNumber": {
"analyzer": "custom_keyword",
"type": "string"
},
"RequisitionNumber": {
"analyzer": "custom_keyword",
"type": "string"
},
"ReceiverOrgCountryCode": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"FlowStatus": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"Name": {
"type": "string"
},
"InboundGSSender": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundAdapterId": {
"type": "long"
},
"ProcessingMode": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"CurrentEventName": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"CurrentActivityStage": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"GSSender": {
"analyzer": "custom_keyword",
"type": "string"
},
"POLocator": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundBillTo": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundBillToEID": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundBillToQualifier": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundShipTo": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundShipToEID": {
"analyzer": "custom_keyword",
"type": "string"
},
"InboundShipToQualifier": {
"analyzer": "custom_keyword",
"type": "string"
},
"VisibilitySenderOrgId": {
"type": "long"
},
"VisibilityReceiverOrgId": {
"type": "long"
},
"VisibilitySenderEID": {
"type": "string"
},
"VisibilityReceiverEID": {
"type": "string"
},
"VisibilitySenderRootOrgEID": {
"type": "string"
},
"VisibilityReceiverRootOrgEID": {
"type": "string"
},
"MRS_INBOUND_INTEGRATION": {
"type": "string"
},
"DOC_STORE_INBOUND_INTEGRATION": {
"type": "string"
},
"APRIME_INBOUND_INTEGRATION": {
"type": "string"
},
"DOC_STORE_EU_INBOUND_INTEGRATION": {
"type": "string"
},
"MRS_OUTBOUND_INTEGRATION": {
"type": "string"
},
"DOC_STORE_OUTBOUND_INTEGRATION": {
"type": "string"
},
"AccountNumber": {
"type": "string"
},
"LatestFileSize": {
"type": "long"
},
"CorrelatedFlowId": {
"type": "keyword"
},
"RemoteArrivalTimeStamp": {
"format": "epoch_millis",
"type": "date"
},
"RemoteSendTimeStamp": {
"format": "epoch_millis",
"type": "date"
},
"Services": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"LastErrorEvent": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"ResolvedBy": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"ResolveReason": {
"analyzer": "whitespace",
"type": "text"
},
"LastErrorMessage": {
"analyzer": "whitespace",
"type": "text"
},
"B2BSiteCode": {
"analyzer": "custom_keyword",
"type": "string"
},
"BuyerId": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"SourceDocumentId": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"PurchaseOrderTypeCode": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
},
"PayerId": {
"analyzer": "custom_keyword",
"type": "text"
},
"BusinessProcessType": {
"type": "keyword",
"normalizer": "lowercase_normalizer"
}
},
"_all": {
"enabled": false
}
}
},
"aliases": {
"corex_activityflow_all": {}
}
}


devint-newrelic-cpu-template.json:

{
"order": 0,
"template": "devint-newrelic-cpu-utilization*",
"settings": {},
"mappings": {
"system": {
"properties": {
"user_p": {
"type": "float"
},
"hostname": {
"type": "keyword"
},
"application": {
"type": "keyword"
}
}
}
},
"aliases": {
"devint-newrelic-cpu-utilization_all": {}
}
}



ES-template with ttl:

{
"order": 0,
"template": "cpu-utilization*",
"settings": {},
"mappings": {
"system": {
"_ttl": {
"default": "3d",
"enabled": true
},
"properties": {
"user_p": {
"type": "float"
},
"hostname": {
"analyzer": "keyword",
"type": "string"
},
"application": {
"analyzer": "keyword",
"type": "string"
}
}
}
},
"aliases": {
"cpu-utilization_all": {}
}
}


Export All File:

$scope.exportData = function () {
var blob = new Blob([convertToCsv($scope.testResults)], { type: 'text/csv' });
saveAs(blob, 'runHistoryReport.csv');
};

function convertToCsv(testResultItem) {
var items = testResultItem;
var itemtab = '\t\t\t\t\t\t';
var firstHeaders = angular.uppercase(localization.get('testframework.runHistory.runName') + ',' +
localization.get('label.action') + ',' +
localization.get('testframework.runHistory.runStatus') + ',' + localization.get('testframework.savedTestName') + ',' +
localization.get('label.scenarioType') + ',' + localization.get('testframework.runHistory.clusterInformation') + ',' +
localization.get('testframework.runHistory.snapshot') + ',' + localization.get('js.general.column.createdBy') + ',\t' +
localization.get('testframework.runHistory.createdDate') + ',' + localization.get('js.general.column.updatedBy') + ',\t' +
localization.get('testframework.runHistory.updatedDate') + ',' + localization.get('testframework.runHistory.setupName') + ',' +
localization.get('testframework.home.configurationDetails') + ',' + itemtab +
localization.get('testframework.home.testCaseDetails') + ',' + '\n');
var firstHeaderitem = firstHeaders + items.map(function (testResult) {
var firstHeaderAndItemResult = '';
if (angular.isDefined(testResult)) {
var testCaseNames = [];

if (testResult.runConfig) {
angular.forEach(testResult.runConfig.testCases, function (testCas) {
if (!testCaseNames.includes(testCas.name)) {
testCaseNames.push('testCaseName:' + testCas.name + ',ConnectionName:' + testCas.connection.name);
}
angular.forEach(testCas.documentId, function (testCaseDoc) {
testCaseNames.push(',Doc Type:' + testCaseDoc.docType + 'Doc Sub Type:' + testCaseDoc.docSubType + 'Doc Version:' + testCaseDoc.version + ',');
});
});
}
var testcases = '';

if (testCaseNames.length > 0) {
angular.forEach(testCaseNames, function (testCaseName) {
testcases += testCaseName;
});
}

var testRunConfigDetails = '';
var testRunConfigName = '';

if (testResult.runConfig) {
testRunConfigName = testResult.runConfig.testRunConfigName;
testRunConfigDetails = 'testRunConfigName:' + testResult.runConfig.testRunConfigName + ',minionsCount:' + testResult.runConfig.minionsCount + ',duration:' + testResult.runConfig.duration +
',durationType:' + testResult.runConfig.durationType + ',rampup:' + testResult.runConfig.rampup + ',rampupType:' + testResult.runConfig.rampupType + ',runSettings:' + testResult.runConfig.runSettings;
}

var testCaselist = 0;

if (testCaseNames.length > testCaselist) {
$scope.testCaseLength = testCaseNames.length;
}

var saveTestName = (testResult.runStatus === 'Completed' || testResult.runStatus === 'Failed' || testResult.runStatus === 'Stopped') ? testResult.savedTestName : '';

firstHeaderAndItemResult = testResult.testRunName + ',' +
testResult.notes + ',' + testResult.runStatus + ',' + saveTestName + ',' +
testResult.type + ',' + testResult.clusterInfo + ',' +
testResult.esSnapshotReference + ',' + testResult.created.by + ',\t' + testResult.created.on + ',' +
testResult.updated.by + ',\t' +
testResult.updated.on + ',' + testRunConfigName + ',' +
testRunConfigDetails + ',' + testcases + ',';
}

return firstHeaderAndItemResult;
}).join('\n');
var secondHeaders = angular.uppercase('\n\n' +
localization.get('testframework.runHistory.jmeterRunTime') +
localization.get('testframework.runHistory.hrs') + ',' + localization.get('testframework.runHistory.jmeterAvgDocUploadTime') +
localization.get('testframework.runHistory.hrs') + ',' + localization.get('testframework.runHistory.jmeterDocUpload') +
localization.get('testframework.runHistory.uploadSec') + ',' + localization.get('testframework.runHistory.jmeterUploadFailure') + ',' +
localization.get('testframework.runHistory.corexRunTime') + localization.get('testframework.runHistory.hrs') + ',' +
localization.get('testframework.runHistory.corexAvgDocProcessingTime') + localization.get('testframework.runHistory.sec') + ',' +
localization.get('testframework.runHistory.corexDoc') + localization.get('testframework.runHistory.processedSec') + ',' +
localization.get('testframework.runHistory.corexDocMetricsTotal') + ',' + localization.get('testframework.runHistory.corexMetricsSuccess') + ',' +
localization.get('testframework.runHistory.corexMetricsResolved') + ',' + localization.get('testframework.runHistory.corexMetricsInprogress') + ',' +
localization.get('testframework.runHistory.corexMetricsError') + '\n');

var secondHeaderitem = secondHeaders + items.map(function (testResult) {
var secondHeaderAndItemResult = '';
if (angular.isDefined(testResult)) {
var testResultDetails = '';
angular.forEach(testResult.testResults, function (testRes) {
testResultDetails = testRes.jmeterActualProcessingTime + ',' + testRes.jmeterAvgResponseTime + ',' + testRes.jmeterThroughput + ',' + testRes.jmeterError + ',' +
testRes.egxActualProcessingTime + ',' + testRes.egxAvgResponseTime + ',' + testRes.egxThroughput + ',' + testRes.egxTotal + ',' + testRes.egxSuccess + ',' +
testRes.egxResolved + ',' + testRes.egxInProgress + ',' + testRes.egxError + ',';
});

secondHeaderAndItemResult = testResultDetails + ',';
}
return secondHeaderAndItemResult;
}).join('\n');

return firstHeaderitem + secondHeaderitem;
}


Bar and Pie chart:

getActivityMetricsPieChartOptions: function () {
return {
chart: {
type: 'pieChart',
showLabels: false,
height: 400,
x: function (d) {
return d.key;
},
y: function (d) {
return parseFloat(d.value);
},
duration: 0,
showLegend: false,
},
};
},

getActivityMetricsBarChartOptions: function () {
return {
chart: {
type: 'discreteBarChart',
height: 450,
margin: {
top: 20,
right: 100,
bottom: 200,
left: 80,
},
x: function(d) {
return d.key;
},
y: function(d) {
return parseFloat(d.value);
},
valueFormat: function(d) {
return d3.format(',.4f')(d);
},
duration: 0,
xAxis: {
axisLabel: localization.get('testframework.activities'),
rotateLabels: 50,
},
yAxis: {
axisLabel: localization.get('testframework.values') +
' (' + localization.get('label.seconds') + ')',
axisLabelDistance: 15,
},
//this makes initial rendering faster especially in case you have a lot of data
transitionDuration: 500,
},
};
},

<div class="row col-sm-7 table-view-padding">
<table ng-show="tableView" class="table">
<tr>
<th l10n="testframework.status"></th>
<th l10n="testframework.expected"></th>
<th l10n="testframework.actual"></th>
<th ng-show="activeTest.replayOnNewEnvironment" l10n="testframework.discrepancy"></th>
</tr>
<tr>
<td l10n="testframework.success"></td>
<td>
<span ng-show="activeTest.replayOnNewEnvironment">{{activeTest.replayRunConfig.documentStatistics.successDocsCount}}></span>
<span ng-show="!activeTest.replayOnNewEnvironment">{{testDetails.jmeterSuccess}}</span>
</td>
<td>{{testDetails.egxSuccess}}</td>
<td ng-show="activeTest.replayOnNewEnvironment">
<a href ng-click="showDiscrepancyForTestRun(activeTest, 'success')">
<i class="fa fa-eye" aria-hidden="false"></i>
</a>
</td>
</tr>
<tr>
<td l10n="testframework.error"></td>
<td>
<span ng-show="activeTest.replayOnNewEnvironment">{{activeTest.replayRunConfig.documentStatistics.failedDocsCount}}></span>
<span ng-show="!activeTest.replayOnNewEnvironment">0</span>
</td>
<td>{{testDetails.egxError}}</td>
<td ng-show="activeTest.replayOnNewEnvironment">
<a href ng-click="showDiscrepancyForTestRun(activeTest, 'error')">
<i class="fa fa-eye" aria-hidden="false"></i>
</a>
</td>
</tr>
<tr>
<td l10n="testframework.resolved"></td>
<td>
<span ng-show="activeTest.replayOnNewEnvironment">{{activeTest.replayRunConfig.documentStatistics.resolvedDocsCount}}></span>
<span ng-show="!activeTest.replayOnNewEnvironment">0</span>
</td>
<td>{{testDetails.egxResolved}}</td>
<td ng-show="activeTest.replayOnNewEnvironment">
<a href ng-click="showDiscrepancyForTestRun(activeTest, 'resolved')">
<i class="fa fa-eye" aria-hidden="false"></i>
</a>
</td>
</tr>
<tr>
<td l10n="testframework.inProgress"></td>
<td>
<span ng-show="activeTest.replayOnNewEnvironment">{{activeTest.replayRunConfig.documentStatistics.inProgressDocsCount}}></span>
<span ng-show="!activeTest.replayOnNewEnvironment">0</span>
</td>
<td>{{testDetails.egxInProgress}}</td>
<td ng-show="activeTest.replayOnNewEnvironment">
<a href ng-click="showDiscrepancyForTestRun(activeTest, 'in_progress')">
<i class="fa fa-eye" aria-hidden="false"></i>
</a>
</td>
</tr>
<tr>
<td l10n="testframework.total"></td>
<td>
<span ng-show="activeTest.replayOnNewEnvironment">{{activeTest.replayRunConfig.status.totalDocs}}></span>
<span ng-show="!activeTest.replayOnNewEnvironment">{{testDetails.jmeterSuccess}}</span>
</td>
<td colspan="2">{{testDetails.egxTotal}}</td>
</tr>
</table>
<div class="row" ng-show="barView">
<div class="col-sm-6 svg-bar-pie-align" ng-if="compareMode">
                    <div>{{::'testframework.total' | l10n}} : {{testDetails.egxTotal}}</div>
<nvd3 options="barChartForMetrics" data="barChartOfCurrentTest"></nvd3>
</div>
</div>
<div ng-if='pieView'>
                <div>{{::'testframework.total' | l10n}} : {{testDetails.egxTotal}}</div>
<nvd3 width="350" height="350" options="pieChartForMetrics" data="pieChartOfCurrentTest"></nvd3>
</div>
</div>
</div>

$scope.tableChart = function () {
$scope.tableView = true;
$scope.barView = false;
$scope.pieView = false;
};

$scope.barChartset = function (testDetails, comparedTestDetails) {
$scope.tableView = false;
$scope.barView = true;
$scope.pieView = false;

$scope.barChartOfComparedTest = [
!$scope.validateData($scope.comparedTestDetails) ?
{
'key': 'comparedTestRunDetails',
'values': [
{ 'label': localization.get('testframework.success'),
'value': $scope.validateData(comparedTestDetails.egxSuccess) ? 0 : comparedTestDetails.egxSuccess},
{ 'label': localization.get('testframework.error'),
'value': $scope.validateData(comparedTestDetails.egxError) ? 0 : comparedTestDetails.egxError},
{ 'label': localization.get('testframework.resolved'),
'value': $scope.validateData(comparedTestDetails.egxResolved) ? 0 : comparedTestDetails.egxResolved},
{ 'label': localization.get('testframework.inProgress'),
'value': $scope.validateData(comparedTestDetails.egxInProgress) ? 0 : comparedTestDetails.egxInProgress}
]
} :
{
'key': 'comparedTestRunDetails',
'values': [
{
'label': '',
'value': 0
}
]
}
];

$scope.barChartOfCurrentTest = [
!$scope.validateData($scope.testDetails) ?
{
'key': 'TestRunDetails',
'values': [
{ 'label': localization.get('testframework.success'),
'value': $scope.validateData(testDetails.egxSuccess) ? 0 : testDetails.egxSuccess},
{ 'label': localization.get('testframework.error'),
'value': $scope.validateData(testDetails.egxError) ? 0 : testDetails.egxError},
{ 'label': localization.get('testframework.resolved'),
'value': $scope.validateData(testDetails.egxResolved) ? 0 : testDetails.egxResolved},
{ 'label': localization.get('testframework.inProgress'),
'value': $scope.validateData(testDetails.egxInProgress) ? 0 : testDetails.egxInProgress}
]
} :
{
'key': 'TestRunDetails',
'values': [
{
'label': '',
'value': 0
}
]
}
];
};

$scope.pieChartset = function (testDetails, comparedTestDetails) {
$scope.tableView = false;
$scope.barView = false;
$scope.pieView = true;

$scope.pieChartOfCurrentTest =
!$scope.validateData($scope.testDetails) ?
[
{ key: localization.get('testframework.success'),
value: $scope.validateData(testDetails.egxSuccess) ? 0 : testDetails.egxSuccess,
total: testDetails.egxTotal
},
{ key: localization.get('testframework.error'),
value: $scope.validateData(testDetails.egxError) ? 0 : testDetails.egxError,
total: testDetails.egxTotal
},
{ key: localization.get('testframework.resolved'),
value: $scope.validateData(testDetails.egxResolved) ? 0 : testDetails.egxResolved,
total: testDetails.egxTotal
},
{ key: localization.get('testframework.inProgress'),
value: $scope.validateData(testDetails.egxInProgress) ? 0 : testDetails.egxInProgress,
total: testDetails.egxTotal
}
] :
[
key: 'TestRunDetails',
value: 0
}
];

$scope.pieChartOfComparedTest =
!$scope.validateData($scope.comparedTestDetails) ?
[
{ key: localization.get('testframework.success'),
value: $scope.validateData(comparedTestDetails.egxSuccess) ? 0 : comparedTestDetails.egxSuccess,
total: comparedTestDetails.egxTotal
},
{ key: localization.get('testframework.error'),
value: $scope.validateData(comparedTestDetails.egxError) ? 0 : comparedTestDetails.egxError,
total: comparedTestDetails.egxTotal
},
{ key: localization.get('testframework.resolved'),
value: $scope.validateData(comparedTestDetails.egxResolved) ? 0 : comparedTestDetails.egxResolved,
total: comparedTestDetails.egxTotal
},
{ key: localization.get('testframework.inProgress'),
value: $scope.validateData(comparedTestDetails.egxInProgress) ? 0 : comparedTestDetails.egxInProgress,
total: comparedTestDetails.egxTotal
}
] :
[
key: 'comparedTestRunDetails',
value: 0
}
];
};

$scope.barChartForMetrics = {
chart: {
type: 'discreteBarChart',
height: 350,
width: 400,
x: function(d) {
return d.label;
},
y: function(d) {
return d.value;
},
showValues: true,
valueFormat: function(d) {
return d;
},
duration: 500,
xAxis: {
axisLabel: 'Jmeter Metrics Status'
},
yAxis: {
axisLabel: 'Values',
axisLabelDistance: -10,
},
transitionDuration: 500
}
};

$scope.pieChartForMetrics = {
chart: {
type: 'pieChart',
height: 500,
x: function(d) {
return d.key + ' - ' + (d.value * 100 / d.total).toFixed(2) + '%';
},
y: function(d) {
return d.value;
},
showLabels: true,
labelType: 'key',
labelsOutside: false,
duration: 0,
labelThreshold: 0.01,
labelSunbeamLayout: true,
legend: {
margin: {
top: 15,
right: 35,
bottom: 5,
left: 10
}
}
}
};

}

<div class="row docflow-graphview-padding">
<i class="fa fa-table" id="docFlowCompareTableIcon"></i>
<input type="button"
       class="btn btn-sm btn-link"
       ng-style="{'background-color': tableView ? 'lightgrey' : ''}"
       value="{{::'testframework.table' | l10n}}"
       ng-click="tableChart()">
<i class="fa fa-bar-chart" id="docFlowCompareBarIcon"></i>
<input type="button"
       class="btn btn-sm btn-link"
       ng-style="{'background-color': barView ? 'lightgrey' : ''}"
       value="{{::'testframework.barChart' | l10n}}"
       ng-click="barChartset(testDetails,comparedTestDetails)">
<i class="fa fa-pie-chart" id="docFlowComparePieIcon"></i>
<input type="button"
       class="btn btn-sm btn-link"
       ng-style="{'background-color': pieView ? 'lightgrey' : ''}"
       value="{{::'testframework.pieChart' | l10n}}"
       ng-click="pieChartset(testDetails,comparedTestDetails)">
</div>


Export jsPDF:

<button type="button"
        id="exportComparison"
        class="btn btn-primary"
        ng-if="compareMode"
        ng-click="exportTestResultsComparisonData('exportCompare')">
{{exportCompareResultLabel}}
</button>

$scope.exportTestResultsComparisonData = function (dataId) {
$scope.exportCompareResultLabel = localization.get('testframework.exportComparison.downloadingLabel');
$scope.updateLoader(true);
try {
domtoimage.toPng(document.getElementById(dataId))
.then(function (blob) {
var pdf = new jsPDF('p', 'mm', 'a4');
var width = pdf.internal.pageSize.getWidth();
var height = pdf.internal.pageSize.getHeight();
pdf.addImage(blob, 'PNG', 0, 0, width, height);
pdf.save('compareMetrics.pdf');
$scope.exportCompareResultLabel = localization.get('testframework.testresults.comparisondata.export');
$scope.updateLoader(false);
});
} catch (error) {
appToaster.pop(localization.get('testframework.exportfailed'));
}
};


Newrelic with api URL connection:

#!/usr/bin/python
"""
collects cpu metric details and push to ES
"""
import logging
import time
import json
import re
from datetime import datetime, timedelta
import boto3
import requests
from requests import get
from requests.utils import quote
from elasticsearch import Elasticsearch, helpers

def load_config(configobj, cfile):
    """load configuration details"""
    f = open(cfile, 'r')
    for line in f:
        line = line.strip()
        (param, value) = re.split(':', line, maxsplit=1)
        configobj[param] = value
    f.close()
    return configobj

# end load_config

def set_subdomain(metaobj):
    """set sub domain details"""
    if metaobj['accountId'] == "050190852521":
        subd = 'awsdsi.ghx.com'
    else:
        subd = 'awsprd.ghx.com'
    return subd

# end set_subdomain

def get_application_name(servicename):
    """get application name"""
    applications = {
        "corex-dp": env.upper() + "-data-poller",
        "corex-processing": env.upper() + "-processing-engine",
        "corex-io": env.upper() + "-io-engine",
        "corex-jb": env.upper() + "-jobs",
        "corex-regapi": env.upper() + "-regapi",
    }
    return applications.get(servicename, servicename)

# end get application name

dateTag = datetime.now().strftime("%Y-%b-%d")
LOG_FILENAME = '/var/log/cpumetric/cpunewrelicmetriccollection.log_%s.log'
logging.basicConfig(filename=LOG_FILENAME % dateTag, level=logging.INFO)
logging.info('Started')

configuration = {}
load_config(configuration, '/opt/conf/cf-params.txt')
application = configuration['Application']

meta = json.loads(get('http://169.254.169.254/latest/dynamic/instance-identity/document/').content)

subdomain = set_subdomain(meta)

es = Elasticsearch(['https://' + meta['region'] + '-testframework-es.' + subdomain ] , verify_certs=False)
cw = boto3.resource('ec2', region_name=meta['region'])
total_exec_time = 0
cpuutilizations = []
env = configuration['GHXEnvironment'].lower()

newrelic_api_insight_url = configuration['NewrelicApiInsightUrl']
newrelic_query_key = configuration['NewrelicQueryKey']
indexname = env + '-newrelic-' + 'cpu-utilization'
max_indexname = env + '-newrelic-' + 'max-cpu-utilization'
while True:
    try:
        cpuutilization = {}
        max_cpuutilization = {}
        now = datetime.utcnow()
        past = (now - timedelta(minutes=1)).strftime("%Y-%m-%dT%H:%M:%S") + "+00:00"
        nrql_past = (now - timedelta(minutes=1)).strftime("%Y-%m-%d %H:%M:%S")
        nrql_future = now.strftime("%Y-%m-%d %H:%M:%S")
        now_date_time = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") + "+00:00"
        exec_starttime = datetime.utcnow()
        nrql_ds_cpu_es_url = newrelic_api_insight_url + "query?nrql=SELECT average(cpuPercent)," \
                             "max(cpuPercent) " \
                             "FROM SystemSample " \
                             "WHERE label.environment='" + env.upper() + "' AND " \
                             "label.service in " \
                             "('corex-es','corex-dp','corex-processing','corex-io','corex-jb'," \
                             "'corex-regapi') " \
                             "SINCE '" + nrql_past + "' UNTIL '" + nrql_future + "' " \
                             "FACET label.service,ec2PrivateIpAddress,label.Name LIMIT 70"
        nrql_ds_cpu_es_results = requests.get(nrql_ds_cpu_es_url,
                                              headers={"X-Query-Key" : newrelic_query_key,
                                                       "Accept": "application/json"})
        nrql_ds_cpu_es_facets = nrql_ds_cpu_es_results.json()['facets']
        for nrql_ds_cpu_es_facet in nrql_ds_cpu_es_facets:
            application_name = ""
            group_name = ""
            if nrql_ds_cpu_es_facet['name'][0] == 'corex-es':
                group_name = nrql_ds_cpu_es_facet['name'][2]
                group_name = group_name.replace('-', '.')
            ip = nrql_ds_cpu_es_facet['name'][1]
            ip = ip.replace('-', '.')
            application_name = get_application_name(nrql_ds_cpu_es_facet['name'][0])
            af_es_datahot = "CoreX." + env.upper() + ".AF.ES.datahot"
            af_es_datawarm = "CoreX." + env.upper() + ".AF.ES.datawarm"
            bt_es_datahot = "CoreX." + env.upper() + ".BT.ES.datahot"
            bt_es_datawarm = "CoreX." + env.upper() + ".BT.ES.datawarm"
            bt_es_datacold = "CoreX." + env.upper() + ".BT.ES.datacold"
            af_es_datacold = "CoreX." + env.upper() + ".AF.ES.datacold"
            af_es_coordinating = "CoreX." + env.upper() + ".AF.ES.coordinating"
            bt_es_coordinating = "CoreX." + env.upper() + ".BT.ES.coordinating"
            af_es_master = "CoreX." + env.upper() + ".AF.ES.master"
            bt_es_master = "CoreX." + env.upper() + ".BT.ES.master"
            es_list = [bt_es_datacold, af_es_datacold, af_es_coordinating, bt_es_coordinating, \
               af_es_master, bt_es_master]
            if group_name not in es_list:
                if af_es_datahot == group_name:
                    application_name = env.upper() + " " + "ES ACTIVITY FLOW"
                    ip = ip + ".AF.ES.datahot"
                elif af_es_datawarm == group_name:
                    application_name = env.upper() + " " + "ES ACTIVITY FLOW"
                    ip = ip + ".AF.ES.datawarm"
                elif bt_es_datahot == group_name:
                    application_name = env.upper() + " " + "ES BUSINESS TRANSACTION"
                    ip = ip + ".BT.ES.datahot"
                elif bt_es_datawarm == group_name:
                    application_name = env.upper() + " " + "ES BUSINESS TRANSACTION"
                    ip = ip + ".BT.ES.datawarm"
                cpuutilization = {"user_p": nrql_ds_cpu_es_facet['results'][0]['average'], \
                                  "hostname": ip, "application": application_name, \
  "@timestamp": now_date_time}
                max_cpuutilization = {"user_p": nrql_ds_cpu_es_facet['results'][1]['max'], \
                                  "hostname": ip, "application": application_name, \
  "@timestamp": now_date_time}
                logging.debug(cpuutilization)
                logging.debug(max_cpuutilization)
                data_dict = {
                    '_op_type': 'index',
                    '_index': indexname,
                    '_source': cpuutilization
                    }
                max_data_dict = {
                    '_op_type': 'index',
                    '_index': max_indexname,
                    '_source': max_cpuutilization
                    }
                cpuutilizations.append(data_dict)
                cpuutilizations.append(max_data_dict)
        try:
            helpers.bulk(client=es, actions=cpuutilizations)
        except Exception as ex:
            logging.exception(ex)
        logging.info('Insert application ES CPU Utilization of Newrelic data into ES')
        es.indices.refresh()
        cpuutilizations = []
        label_env = quote("%Corex-" + env.upper() + "-db%")
        nrql_ds_cpu_db_url = newrelic_api_insight_url + "query?nrql=SELECT average(cpuPercent), " \
                             "max(cpuPercent) " \
                             "FROM SystemSample WHERE ec2Tag_Name LIKE'" + label_env + "' " \
                             "AND label.service='corex-mongo' SINCE '" + nrql_past + "' " \
                             "UNTIL '" + nrql_future + "' " \
                             "FACET label.service,ec2PrivateIpAddress,label.Name LIMIT 50"
        nrql_ds_cpu_db_results = requests.get(nrql_ds_cpu_db_url,
                                              headers={"X-Query-Key" : newrelic_query_key,
                                                       "Accept": "application/json"})
        nrql_ds_cpu_db_facets = nrql_ds_cpu_db_results.json()['facets']
        for nrql_ds_cpu_db_facet in nrql_ds_cpu_db_facets:
            ip = nrql_ds_cpu_db_facet['name'][1]
            ip = ip.replace('-', '.')
            appname = ""
            if ("config" or "archive") not in nrql_ds_cpu_db_facet['name'][2]:
                if "CoreX-"+env.upper()+"-db-audit" in nrql_ds_cpu_db_facet['name'][2]:
                    appname = env.upper() + " " + "MONGO AUDIT"
                elif "CoreX-"+env.upper()+"-db-eb" in nrql_ds_cpu_db_facet['name'][2]:
                    appname = env.upper() + " " + "MONGO EB EXECUTION"
                elif ("CoreX-"+env.upper()+"-db-bt") in nrql_ds_cpu_db_facet['name'][2]:
                    appname = env.upper() + " " +  "MONGO BUSINESS TRANSACTION"
                elif ("Corex-"+env.upper()+"-db-bt") in nrql_ds_cpu_db_facet['name'][2]:
                    appname = env.upper() + " " +  "MONGO BUSINESS TRANSACTION"
                cpuutilization = {"user_p": nrql_ds_cpu_db_facet['results'][0]['average'],
                                  "hostname": ip, "application": appname,
                                  "@timestamp": past}
                max_cpuutilization = {"user_p": nrql_ds_cpu_db_facet['results'][1]['max'], \
                                  "hostname": ip, "application": appname, \
  "@timestamp": past}
            logging.debug(cpuutilization)
            data_dict = {
                '_op_type': 'index',
                '_index': indexname,
                '_source': cpuutilization
                }
            max_data_dict = {
                '_op_type': 'index',
                '_index': max_indexname,
                '_source': max_cpuutilization
                }
            if appname:
                cpuutilizations.append(data_dict)
                cpuutilizations.append(max_data_dict)
        try:
            helpers.bulk(client=es, actions=cpuutilizations)
        except Exception as ex:
            logging.exception(ex)
        es.indices.refresh()
        cpuutilizations = []
        exec_endtime = datetime.utcnow()
        total_exec_time = abs(exec_endtime-exec_starttime).seconds
    except Exception as ex:
        logging.exception("message")
    finally:
        time.sleep(35 - total_exec_time)
logging.info('End')


Newrelic with Insight URL connection:

#!/usr/bin/python
"""
collects heap metric details and push to ES
"""
import logging
import time
import json
import re
from datetime import datetime, timedelta
import requests
from requests import get
from requests.utils import quote
from elasticsearch import Elasticsearch, helpers


def load_config(configobj, cfile):
    """load configuration details"""
    f = open(cfile, 'r')
    for line in f:
        line = line.strip()
        (param, value) = re.split(':', line, maxsplit=1)
        configobj[param] = value
    f.close()
    return configobj

# end load_config

def set_subdomain(metaobj):
    """set sub domain details"""
    if metaobj['accountId'] == "050190852521":
        subd = 'awsdsi.ghx.com'
    else:
        subd = 'awsprd.ghx.com'
    return subd

# end set_subdomain

def get_ip(hosts_details, application_host_detail):
    """get ip"""
    for host in hosts_details:
        if host['id'] == application_host_detail:
            host_ip_detail = host['host']
            return host_ip_detail

# end get_ip

def get_application_name(servicename):
    """get application name"""
    applications = {
        "corex-dp": env.upper() + "-data-poller",
        "corex-processing": env.upper() + "-processing-engine",
        "corex-io": env.upper() + "-io-engine",
        "corex-jb": env.upper() + "-jobs",
    }
    return applications.get(servicename, servicename)

# end get application name

dateTag = datetime.now().strftime("%Y-%b-%d")
LOG_FILENAME = '/var/log/heapmetric/heapnewrelicmetriccollection.log_%s.log'
logging.basicConfig(filename=LOG_FILENAME % dateTag, level=logging.INFO)
logging.info('Started')

configuration = {}
load_config(configuration, '/opt/conf/cf-params.txt')
application = configuration['Application']

meta = json.loads(get('http://169.254.169.254/latest/dynamic/instance-identity/document/').content)
instance_id = meta['instanceId']
hostname = meta['privateIp']

subdomain = set_subdomain(meta)
es = Elasticsearch(['https://' + meta['region'] + '-testframework-es.' + subdomain ] , verify_certs=False)

total_exec_time = 0
heaputilizations = []
env = configuration['GHXEnvironment'].lower()

newrelic_api_insight_url = configuration['NewrelicApiInsightUrl']
newrelic_query_key = configuration['NewrelicQueryKey']
indexname = env + '-newrelic-' + 'heap-utilization'
while True:
    try:
        heaputilization = {}
        now = datetime.utcnow()
        past = (now - timedelta(minutes=1)).strftime("%Y-%m-%dT%H:%M:%S") + "+00:00"
        future = now.strftime("%Y-%m-%dT%H:%M:%S") + "+00:00"
        nrql_past = (now - timedelta(minutes=1)).strftime("%Y-%m-%d %H:%M:%S")
        nrql_future = now.strftime("%Y-%m-%d %H:%M:%S")
        exec_starttime = datetime.utcnow()
        nrql_heap_url = newrelic_api_insight_url + "query?nrql=" \
                          "SELECT average(memoryVirtualSizeBytes) FROM ProcessSample where " \
                          "label.environment='" + env.upper() + "' AND label.service " \
                          "in ('corex-io','corex-dp','corex-processing','corex-jb') AND " \
                          "processDisplayName='java' " \
                          "SINCE '" + nrql_past + "' UNTIL '" + \
                          nrql_future + "' FACET label.service, ec2PrivateIpAddress,label.Name," \
                          "hostname LIMIT 50"
        nrql_heap_results = requests.get(nrql_heap_url,
                                         headers={"X-Query-Key": newrelic_query_key,
                                                  "Accept": "application/json"})
        nrql_heap_facets = nrql_heap_results.json()['facets']
        for nrql_heap_facet in nrql_heap_facets:
            ip = nrql_heap_facet['name'][1]
            ip = ip.replace('-', '.')
            memoryBytes = (float)(nrql_heap_facet['results'][0]['average'])
            application_name = get_application_name(nrql_heap_facet['name'][0])
            heaputilization = {"used_bytes_by_host": memoryBytes,
                               "hostname": ip, "application": application_name,
                               "@timestamp": future}
            logging.debug(heaputilizations)
            data_dict = {
                '_op_type': 'index',
                '_index': indexname,
                '_source': heaputilization
            }
            heaputilizations.append(data_dict)
        helpers.bulk(client=es, actions=heaputilizations)
        logging.info('Insert engines,dp,jobs Heap Utilization of Newrelic data into ES')
        try:
            es.indices.refresh()
        except Exception as ex:
            logging.exception(ex)
        heaputilizations = []
        nrql_ds_heap_es_url = newrelic_api_insight_url + "query?nrql=" \
                          "SELECT average(memoryVirtualSizeBytes) FROM ProcessSample where " \
                          "label.environment='" + env.upper() + "' AND label.service " \
                          "='corex-es' AND processDisplayName ='elasticsearch' " \
                          "SINCE '" + nrql_past + "' UNTIL '" + \
                          nrql_future + "' FACET label.service,ec2PrivateIpAddress,label.Name " \
                          "LIMIT 50"
        nrql_ds_heap_es_results = requests.get(nrql_ds_heap_es_url,
                                               headers={"X-Query-Key": newrelic_query_key,
                                                        "Accept": "application/json"})
        nrql_ds_heap_es_facets = nrql_ds_heap_es_results.json()['facets']
        for nrql_ds_heap_es_facet in nrql_ds_heap_es_facets:
            application_name = ""
            group_name = ""
            if nrql_ds_heap_es_facet['name'][0] == 'corex-es':
                group_name = nrql_ds_heap_es_facet['name'][2]
                group_name = group_name.replace('-', '.')
            ip = nrql_ds_heap_es_facet['name'][1]
            ip = ip.replace('-', '.')
            memoryBytes = (float)(nrql_ds_heap_es_facet['results'][0]['average'])
            af_es_datahot = "CoreX." + env.upper() + ".AF.ES.datahot"
            af_es_datawarm = "CoreX." + env.upper() + ".AF.ES.datawarm"
            bt_es_datahot = "CoreX." + env.upper() + ".BT.ES.datahot"
            bt_es_datawarm = "CoreX." + env.upper() + ".BT.ES.datawarm"
            bt_es_datacold = "CoreX." + env.upper() + ".BT.ES.datacold"
            af_es_datacold = "CoreX." + env.upper() + ".AF.ES.datacold"
            af_es_coordinating = "CoreX." + env.upper() + ".AF.ES.coordinating"
            bt_es_coordinating = "CoreX." + env.upper() + ".BT.ES.coordinating"
            af_es_master = "CoreX." + env.upper() + ".AF.ES.master"
            bt_es_master = "CoreX." + env.upper() + ".BT.ES.master"
            es_list = [bt_es_datacold, af_es_datacold, af_es_coordinating, bt_es_coordinating, \
                       af_es_master, bt_es_master]
            if group_name not in es_list:
                if af_es_datahot == group_name:
                    application_name = env.upper() + " " + "ES ACTIVITY FLOW"
                    ip = ip + ".AF.ES.datahot"
                elif af_es_datawarm == group_name:
                    application_name = env.upper() + " " + "ES ACTIVITY FLOW"
                    ip = ip + ".AF.ES.datawarm"
                elif bt_es_datahot == group_name:
                    application_name = env.upper() + " " + "ES BUSINESS TRANSACTION"
                    ip = ip + ".BT.ES.datahot"
                elif bt_es_datawarm == group_name:
                    application_name = env.upper() + " " + "ES BUSINESS TRANSACTION"
                    ip = ip + ".BT.ES.datawarm"
                print(application_name)
                heaputilization = {"used_bytes_by_host": memoryBytes,
                                   "hostname": ip, "application": application_name,
                                   "@timestamp": future}
                logging.debug(heaputilization)
                data_dict = {
                    '_op_type': 'index',
                    '_index': indexname,
                    '_source': heaputilization
                }
                heaputilizations.append(data_dict)
        helpers.bulk(client=es, actions=heaputilizations)
        logging.info('Insert application ES Heap Utilization of Newrelic data into ES')
        try:
            es.indices.refresh()
        except Exception as ex:
            logging.exception(ex)
        heaputilizations = []
        label_env = quote("%Corex-" + env.upper() + "-db%")
        nrql_ds_heap_db_url = newrelic_api_insight_url + "query?nrql=" \
                          "SELECT average(memoryVirtualSizeBytes) FROM ProcessSample where " \
                          "ec2Tag_Name LIKE'" + label_env + "' AND label.service " \
                          "='corex-mongo' AND processDisplayName ='mongod' " \
                          "SINCE '" + nrql_past + "' UNTIL '" + \
                          nrql_future + "' FACET label.service,ec2PrivateIpAddress,label.Name " \
                          "LIMIT 50"
        nrql_ds_heap_db_results = requests.get(nrql_ds_heap_db_url,
                                               headers={"X-Query-Key": newrelic_query_key,
                                                        "Accept": "application/json"})
        nrql_ds_heap_db_facets = nrql_ds_heap_db_results.json()['facets']
        for nrql_ds_heap_db_facet in nrql_ds_heap_db_facets:
            ip = nrql_ds_heap_db_facet['name'][1]
            ip = ip.replace('-', '.')
            appname = ""
            if ("config" or "archive") not in nrql_ds_heap_db_facet['name'][2]:
                if "CoreX-"+env.upper()+"-db-audit" in nrql_ds_heap_db_facet['name'][2]:
                    appname = env.upper() + " " + "MONGO AUDIT"
                elif "CoreX-"+env.upper()+"-db-eb" in nrql_ds_heap_db_facet['name'][2]:
                    appname = env.upper() + " " + "MONGO EB EXECUTION"
                elif ("CoreX-"+env.upper()+"-db-bt") in nrql_ds_heap_db_facet['name'][2]:
                    appname = env.upper() + " " +  "MONGO BUSINESS TRANSACTION"
                elif ("Corex-"+env.upper()+"-db-bt") in nrql_ds_heap_db_facet['name'][2]:
                    appname = env.upper() + " " +  "MONGO BUSINESS TRANSACTION"
                print(appname)
                memoryBytes = (float)(nrql_ds_heap_db_facet['results'][0]['average'])
                heaputilization = {"used_bytes_by_host": memoryBytes,
                                   "hostname": ip, "application": appname,
                                   "@timestamp": future}
            logging.debug(heaputilizations)
            data_dict = {
                '_op_type': 'index',
                '_index': indexname,
                '_source': heaputilization
            }
            if appname:
                heaputilizations.append(data_dict)
        helpers.bulk(client=es, actions=heaputilizations)
        logging.info('Insert DB Heap Utilization of Newrelic data into ES')
        try:
            es.indices.refresh()
        except Exception as ex:
            logging.exception(ex)
        heaputilizations = []
        exec_endtime = datetime.utcnow()
        total_exec_time = abs(exec_endtime - exec_starttime).seconds
    except Exception as ex:
        logging.exception("message")
    finally:
        time.sleep(45 - total_exec_time)
logging.info('End')


cloud mongo metrics script:

#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
collects cloud mongo event bus scan and order metric details and push to ES
"""

import json
import base64
import time
import logging
import requests
import re
from requests import get
from requests.auth import HTTPDigestAuth
from datetime import datetime, timedelta
from elasticsearch import Elasticsearch, helpers
import pymongo
from pymongo import MongoClient

GHX_DEV = '5373a8689194bf134f7a93ad'

configuration = {}


def load_config(configobj, cfile):
    """load configuration details"""

    f = open(cfile, 'r')
    for line in f:
        line = line.strip()
        (param, value) = re.split(':', line, maxsplit=1)
        configobj[param] = value
    f.close()
    return configobj


load_config(configuration, '/opt/conf/cf-params.txt')

environment = configuration['GHXEnvironment'].lower()
cloudmongoapiurl = configuration['CloudMongoApiUrl']

base64_message = configuration['CloudMongoMMSpublickey']
base64_bytes = base64_message.encode('ascii')
message_bytes = base64.b64decode(base64_bytes)
mms_public_key = message_bytes.decode('ascii')

base64_message = configuration['CloudMongoMMSprivatekey']
base64_bytes = base64_message.encode('ascii')
message_bytes = base64.b64decode(base64_bytes)
mms_private_key = message_bytes.decode('ascii')
testResultId = configuration['TestResultId']
testFrameWorkURL = configuration['testFrameWorkURL']

def set_subdomain(metaobj):
    """set sub domain details"""

    if metaobj['accountId'] == '050190852521':
        subd = 'awsdsi.ghx.com'
    else:
        subd = 'awsprd.ghx.com'
    return subd


meta = json.loads(get('http://169.254.169.254/latest/dynamic/'
                  + 'instance-identity/document/').content)
instance_id = meta['instanceId']
hostname = meta['privateIp']

subdomain = set_subdomain(meta)

es = Elasticsearch(['https://' + meta['region'] + '-testframework-es.'
                   + subdomain] , verify_certs=False)

def get_clusters(group_id):
    """ pulls data from mms api - by group"""

    r = requests.get(cloudmongoapiurl + '/' + group_id + '/clusters?',
                     headers={'Accept': 'application/json'},
                     auth=HTTPDigestAuth(mms_public_key,
                     mms_private_key))
    r_json = r.json()
    return r_json['results']


def get_host_with_clusterid(group_id, cluster_id):
    r = requests.get(cloudmongoapiurl + '/' + group_id
                     + '/hosts?clusterId=' + cluster_id,
                     headers={'Accept': 'application/json'},
                     auth=HTTPDigestAuth(mms_public_key,
                     mms_private_key))
    r_json = r.json()
    return r_json['results']


def get_with_meastype_granular_duration(
    href,
    granularity,
    starttime,
    endtime,
    measurementtype,
    hostname,
    ):
    r = requests.get(href + '/measurements?granularity=' + granularity
                     + '&start=' + starttime + '&end=' + endtime + '&m='
                      + measurementtype + '&processId=' + hostname
                     + '&pretty=true',
                     headers={'Accept': 'application/json'},
                     auth=HTTPDigestAuth(mms_public_key,
                     mms_private_key))
    r_json = r.json()
    return r_json['measurements']


count = 0
total_exec_time = 0

dateTag = datetime.now().strftime('%Y-%b-%d')
LOG_FILENAME = \
    '/var/log/cloudmetric/cloudmongometriccollection.log_%s.log'
logging.basicConfig(filename=LOG_FILENAME % dateTag, level=logging.INFO)
logging.info('Started')

uri_mongo = testFrameWorkURL
myclient = pymongo.MongoClient(uri_mongo)
print myclient
mydb = myclient["test-framework"]
collection_andamentos = mydb['MinionCloudMongoCheck']
print collection_andamentos
logging.info(collection_andamentos)
myquery = { 'testScenarioId':testResultId,"collectMinionCloudMetrics": True }
mydoc = collection_andamentos.find(myquery)
secondhit = ''
for x in mydoc:
  print(x)
  secondhit = x
if collection_andamentos.find({'testScenarioId':testResultId}).count() == 0 or secondhit:
    collection_andamentos.insert({'testScenarioId':testResultId,"collectMinionCloudMetrics":True})
    print("inserted")
    logging.info("inserted")
    while True:
        try:
            count += 1
            result_cluster_details = get_clusters(GHX_DEV)
            clusterid = []
            hostnameandhref = {}
            hostreplicawithip = {}

            hostname_with_value_scan_and_order = {}
            hostname_with_opcount_query = {}

            cloudmongoscanandorders = []
            cloudmongoopcounterqueries = []
            now = datetime.utcnow()
            fromdate = (now
                        - timedelta(minutes=1)).strftime('%Y-%m-%dT%H:%M:%S'
                                                         ) + 'Z'
            todate = now.strftime('%Y-%m-%dT%H:%M:%S') + 'Z'
            exec_starttime = datetime.utcnow()
            logging.info(count)
            for (id, element) in enumerate(result_cluster_details):
                if element['clusterName'] == 'event-bus-' + environment \
                    + '-sharded-cluster' and element['typeName'] \
                    == 'REPLICA_SET':
                    clusterid.append(element['id'])
            for clus_id in clusterid:
                result_host_details = get_host_with_clusterid(GHX_DEV,
                    clus_id)
                for element in result_host_details:
                    replicawithip = []
                    replicawithip.append(element['ipAddress'])
                    replicawithip.append(element['replicaSetName'])
                    if element['hostname'] not in hostreplicawithip:
                        hostreplicawithip[element['hostname']] = \
                            replicawithip

                    r = element['hostname']
                    hostnameandhref[r] = element['links']
                    for ref in element['links']:
                        hostnameandhref[r] = ref['href']
            for key in hostnameandhref:
                try:
                    scan_order = []
                    scan_order = get_with_meastype_granular_duration(
                        hostnameandhref[key],
                        'PT10S',
                        fromdate,
                        todate,
                        'OPERATIONS_SCAN_AND_ORDER',
                        key,
                        )
                except Exception, ex:
                    scan_order = []
                    logging.exception(ex)
                for datapoint in scan_order:
                    for timevalue in datapoint['dataPoints']:
                        if isinstance(timevalue['value'], float):
                            cloudmongoscanandorder = {
                                'granularity': 'PT10S',
                                'ipwithhostname': hostreplicawithip[key][0] \
                                    + '-' + key,
                                'clusterName': 'event-bus-' + environment \
                                    + '-sharded-cluster',
                                'replicaset': hostreplicawithip[key][1],
                                'measurement_type': 'OPERATIONS_SCAN_AND_ORDER',
                                'value': timevalue['value'],
                                '@timestamp': timevalue['timestamp'],
                                }
                            logging.info(cloudmongoscanandorder)
                            indexname = environment \
                                + '-cloudmongo-scanandorder'
                            data_dict = {'_op_type': 'index',
                                '_index': indexname,
                                '_source': cloudmongoscanandorder}
                            logging.info(data_dict)
                            cloudmongoscanandorders.append(data_dict)
                try:
                    opcounter = []
                    opcounter = get_with_meastype_granular_duration(
                        hostnameandhref[key],
                        'PT10S',
                        fromdate,
                        todate,
                        'OPCOUNTER_QUERY',
                        key,
                        )
                except Exception, ex:
                    opcounter = []
                    logging.exception(ex)   
                for datapoint in opcounter:

                    for timevalue in datapoint['dataPoints']:
                        if isinstance(timevalue['value'], float):
                            cloudmongoopcounterquery = {
                                'granularity': 'PT10S',
                                'ipwithhostname': hostreplicawithip[key][0] \
                                    + '-' + key,
                                'clusterName': 'event-bus-' + environment \
                                    + '-sharded-cluster',
                                'replicaset': hostreplicawithip[key][1],
                                'measurement_type': 'OPCOUNTER_QUERY',
                                'value': timevalue['value'],
                                '@timestamp': timevalue['timestamp'],
                                }
                            logging.info(cloudmongoopcounterquery)
                            indexname = environment \
                                + '-cloudmongo-opcounter-query'
                            data_dict = {'_op_type': 'index',
                                    '_index': indexname,
                                    '_source': cloudmongoopcounterquery}
                            logging.info(data_dict)
                            cloudmongoopcounterqueries.append(data_dict)
            if count >= 1:
                try:
                    print('start bulk insert')
                    helpers.bulk(client=es, actions=cloudmongoscanandorders)
                    helpers.bulk(client=es, actions=cloudmongoopcounterqueries)
                    print('end bulk insert')
                except Exception, ex:
                    logging.exception(ex)
                    es.indices.refresh()
                    cloudmongoscanandorders = []
                    cloudmongoopcounterqueries = []
                    count = 0
                    exec_endtime = datetime.utcnow()
                    total_exec_time = abs(exec_endtime - exec_starttime).seconds
        except Exception, ex:
            logging.exception('message')
        finally:
            time.sleep(45 - total_exec_time)
            if(len(secondhit) == 0): 
                myquery = {"testScenarioId":testResultId, "collectMinionCloudMetrics": True }
                newvalues = { "$set": { "collectMinionCloudMetrics": False } }
                collection_andamentos.update_one(myquery, newvalues)
                print("inserted false")
    logging.info('End')

how to add sscript to spreadsheet:

using spread sheet to fill form:

function auto_Corex_daily_sheetFill() {
var wrkBK = SpreadsheetApp.getActiveSpreadsheet();
var wrksht = wrkBK.getSheetByName("Sheet1");
var formUrl = "";
var formData = "";
var Emailaddress ="";
var Jira_tickets ="";
var topic_team ="";
var noOfRows = 3;
for(i=2;i<noOfRows;i++){
Emailaddress =wrksht.getRange("A"+i).getValue();
Jira_tickets =wrksht.getRange("B"+i).getValue();
topic_team =wrksht.getRange("C"+i).getValue();
Logger.log(Emailaddress);
Logger.log(Jira_tickets);
Logger.log(topic_team);
var formUrl ="https://docs.google.com/forms/u/0/d/e/1FAIpQLScS-rF83g5Tge1dDswYVU1v7EVLWZAbMlk74ELZ6mDA6OI4dg/formResponse?&pageHistory=0";
var formData ="&emailAddress="+Emailaddress+"&entry.695421166="+Jira_tickets+"&entry.2059349469="+topic_team;
var finalUrl = formUrl+formData;
var options = {
"method": "post"
};
UrlFetchApp.fetch(finalUrl, options);
}
}


go to spread sheet tools add script with script editor, this script will submit value to form with values in spread sheet.

you can keep a button and assign script in click button.




aggregation

createQuery().field(ENGINENAME).equalIgnoreCase(engineName).field("componentId") .equal("eventbus").field("name").equal(name).field("type").equal("NUMERIC").field(TIME) .greaterThanOrEq(Timestamp.valueOf(fromDate)).field(TIME) .lessThanOrEq(Timestamp.valueOf(toDate)) db.getCollection('EBExecutionRunTimeBusinessStatistics').find({"engineName":"ENGINE-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":$gte{Timestamp.valueOf("2021-06-24 04:34:07.584")}},"time":$lte{(Timestamp.valueOf("2021-06-24 04:42:12.195")}}}) .group(Group.id(grouping("month", new Accumulator("$month", TIME)), grouping("day", new Accumulator("$dayOfMonth", TIME)), grouping("year", new Accumulator("$year", TIME)), grouping("hour", new Accumulator("$hour", TIME)), grouping("minutes", new Accumulator("$minute", TIME)), grouping(ENGINENAME, ENGINENAME)), grouping("totalValue", new Accumulator("$sum", "value"))) $group : {"Month" : "$date.month", } .project(projection("month", "_id.month"), projection("day", "_id.day"), projection("year", "_id.year"), projection("hour", "_id.hour"), projection("minutes", "_id.minutes"), projection("engineName", "_id.engineName"), projection("totalValue", "totalValue")) "$project" : { "date" : { "$dateFromString" : { "dateString" : "$createdAt", "onError" : "$date" } }, {$project : { month : {$month : "_id.month"}, year : {$day : "_id.day"}, }}, db.getCollection('EBExecutionRunTimeBusinessStatistics').find({"engineName":"Engine-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":{"$gte":ISODate("2021-06-24 04:34:07.584"),"$lte":ISODate("2021-06-24 04:42:12.195")}}).aggregate({ "$group" : {"_id": {"year" : { "$year" : "time" }, "month" : { "$month" : "time" },"day" : { "$dayOfMonth" : "time" }}}}); { $match : {"engineName":"Engine-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":{"$gte":ISODate("2021-06-24 04:34:07.584"),"$lte":ISODate("2021-06-24 04:42:12.195") } } db.EBExecutionRunTimeBusinessStatisticsAgg.aggregate([{ $match : {"engineName":"Engine-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":{"$gte":ISODate("2021-06-24 04:34:07.584"),"$lte":ISODate("2021-06-24 04:42:12.195") } },{"$group" : {"_id": {"month":{ "$month":"time"}}}}]); db.EBExecutionRunTimeBusinessStatisticsAgg.aggregate([{ $match : {"engineName":"Engine-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":{"$gte":ISODate("2021-06-24 04:34:07.584"),"$lte":ISODate("2021-06-24 04:42:12.195") }}}]) db.EBExecutionRunTimeBusinessStatistics.aggregate([{ $match : {"engineName":"Engine-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":{"$gte":ISODate("2021-06-24 04:34:07.584"),"$lte":ISODate("2021-06-24 04:42:12.195") }}},{"$group":{"_id":{"month":{"$month": "$time"},"day":{"$dayOfMonth":"$time"},"year":{"$year":"$time"},"hour":{"$hour":"$time"},"minutes":{"$minute":"$time"},"totalValue":{"$sum":"$value"}}}}]) db.EBExecutionRunTimeBusinessStatistics.aggregate([ { "$match":{ "engineName":"Engine-PE", "componentId":"eventbus", "name":"bus.publish.events.per.hour", "type":"NUMERIC", "time":{ "$gte":"ISODate(""2021-06-24 04:34:07.584"")", "$lte":"ISODate(""2021-06-24 04:42:12.195"")" } } }, { "$group":{ "_id":{ "month":{ "$month":"$time" }, "day":{ "$dayOfMonth":"$time" }, "year":{ "$year":"$time" }, "hour":{ "$hour":"$time" }, "minutes":{ "$minute":"$time" }, "totalValue":{ "$sum":"$value" } } } } , { "$project":{ "month": "_id.month" }]) db.EBExecutionRunTimeBusinessStatistics.aggregate([{ $match : {"engineName":"Engine-PE","componentId":"eventbus","name":"bus.publish.events.per.hour","type":"NUMERIC","time":{"$gte":ISODate("2021-06-24 04:34:07.584"),"$lte":ISODate("2021-06-24 04:42:12.195") }}},{"$group":{"_id":{"month":{"$month": "$time"},"day":{"$dayOfMonth":"$time"},"year":{"$year":"$time"},"hour":{"$hour":"$time"},"minutes":{"$minute":"$time"},"totalValue":{"$sum":"$value"}}}},{"$project":{"month": "_id.month","day", "_id.day","year", "_id.year","hour", "_id.hour","minutes", "_id.minutes","engineName", "_id.engineName","totalValue", "totalValue"}}])


Query query = createQuery(); query.field(RUN_STATUS).equalIgnoreCase(COMPLETED); query.retrievedFields(true, RELEASE_VERSION); query.order(UPDATED_ON_DESC).asList(); List<TestResult> testResults =getCollection() .distinct(RELEASE_VERSION,query.offset(page).limit(10).getQueryObject());


   List<String> engineNames = Lists.newArrayList();
        engineNames.add("Engine-PE");
        engineNames.add("Engine-DataPoller");
        engineNames.add("Engine-IO");
        LOGGER.info("from date--->"+fromDate+" to date--->"+toDate);
        Iterator<EBExecutionAggregationResult> res = getDatastore().createAggregation(EBExecutionRunTimeBusinessStatisticsAgg.class)
                .match(createQuery().field(ENGINENAME).in(engineNames).field("name").equal(name).field(TIME)
                        .greaterThanOrEq(new Timestamp(fromDate.getTime())).field(TIME).lessThanOrEq(new Timestamp(toDate.getTime())))
                .group(Group.id(grouping(NAME, NAME)), grouping("avgValue", new Accumulator("$avg", "value")))
                .project(projection("name", "_id.name"), projection("value", "avgValue")).aggregate(EBExecutionAggregationResult.class);
        return IteratorUtils.toList(res);


return findBy(Criteria.criteria(Field.field(ENGINENAME).equalIgnoreCase(engine))
.and(Field.field(TIME).gte(fromDate))
.and(Field.field(TIME).lte(toDate))
.and(Field.field("name").equalIgnoreCase(name)).order(Field.field(TIME).asc()).limit(Integer.parseInt(TFConstants.THOUSAND.getValue())));



  LOGGER.debug("engineName:{},name:{},fromDate:{},toDate:{}",engineName,name,fromDate,toDate);
Iterator<EBExecutionRunTimeBusinessStatisticsAggregation> res = getDatastore()
.createAggregation(EBExecutionRunTimeBusinessStatistics.class)
.match(createQuery().field(ENGINENAME).equalIgnoreCase(engineName).field("componentId")
.equal("eventbus").field("name").equal(name).field("type").equal("NUMERIC").field(TIME)
.greaterThanOrEq(fromDate).field(TIME)
.lessThanOrEq(toDate))
.group(Group.id(grouping("month", new Accumulator("$month", TIME)),
grouping("day", new Accumulator("$dayOfMonth", TIME)),
grouping("year", new Accumulator("$year", TIME)),
grouping("hour", new Accumulator("$hour", TIME)),
grouping("minutes", new Accumulator("$minute", TIME)), grouping(ENGINENAME, ENGINENAME)),
grouping("totalValue", new Accumulator("$sum", "value")))
.project(projection("month", "_id.month"), projection("day", "_id.day"), projection("year", "_id.year"),
projection("hour", "_id.hour"), projection("minutes", "_id.minutes"),
projection("engineName", "_id.engineName"), projection("totalValue", "totalValue"))

.aggregate(EBExecutionRunTimeBusinessStatisticsAggregation.class);        
        List<EBExecutionRunTimeBusinessStatisticsAggregation> ebExecutionRunTimeBusinessStatisticsAggregation = Lists.newArrayList();
        res.forEachRemaining(ebExecutionRunTimeBusinessStatisticsAggregation::add);

        if (ebExecutionRunTimeBusinessStatisticsAggregation.size() > 0) {
            LOGGER.debug("ebExecutionRunTimeBusinessStatisticsAggregation.Inside DAO AggTime:{}", ebExecutionRunTimeBusinessStatisticsAggregation);
            return ebExecutionRunTimeBusinessStatisticsAggregation;




<div class="row col-sm-4 testResultName tests-inprogress">{{::'testframework.homePage.testResultName' | l10n}}:
<ui-select ng-model="testResult.selected" theme="bootstrap"  ng-change="updateTestResult()" required>
<ui-select-match class="select-alignment" placeholder="{{::'label.select' | l10n}}">
{{$select.selected.testRunName}}
</ui-select-match>
<ui-select-choices
repeat="testResult in testResults | filter: {'testRunName':$select.search} track by testResult.id">
<span ng-if="testResult.testRunName !==''">{{testResult.testRunName}}</span>
</ui-select-choices>
<ui-select-no-choice> <span>{{::'dashboard.incidentTracker.noMatch' | l10n}} </span>
</ui-select-no-choice>
</ui-select>
</div>


<md-select required name="field" select-infinite-scroll="" ng-model="selected" data-md-container-class="selectdemoSelectHeader">
<md-select-header layout="column" class="demo-select-header">
<section flex>
<span>Header</span>
</section>
<md-input-container>
<input ng-model="selectedRelease.label" type="search" ng-model-options="{debounce: {'default': 1000, 'blur': 0}}"
placeholder="Search..." class="demo-header-searchbox md-text"/>
</md-input-container>
</md-select-header>
<md-optgroup>
<md-option ng-value="release" ng-repeat="release in releaseOptions | filter:selectedRelease.label | limitTo:10">{{release.label}}
{{$last}}
<div ng-if="$last">
<button ng-hide="!loadMore"
    class="btn btn-xs btn-block"
    ng-mouseover="loadMore()"
    ng-disabled="loading">
{{::'customerManagement.orgsManagement.communicationProfiles.loadMore' | l10n}}
</button>
</div>
</md-optgroup>
</md-select>
<div ng-messages="form.field.$error">
<div ng-message="required">Required</div>
</div>
<section>
<md-button ng-click="clear()" class="md-raise">
<i class="material-icons">clear</i>
<md-tooltip>Clear Selection</md-tooltip>
</md-button>
</section>
</md-input-container> 

Comments