diff --git a/CHANGELOG.md b/CHANGELOG.md index 29afe98..13bd1eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Support for JAWS PDU devices - Added support for pprof builds ### Changes diff --git a/Dockerfile b/Dockerfile index af0b5ae..d74ea91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,6 +49,9 @@ ENV API_URL="http://cray-power" ENV API_SERVER_PORT=":28007" ENV API_BASE_PATH="/v1" +ENV PCS_JAWS_MONITOR="true" +ENV PCS_JAWS_MONITOR_INTERVAL="20" + COPY power-control /usr/local/bin/ COPY configs configs diff --git a/Dockerfile.ct.Dockerfile b/Dockerfile.ct.Dockerfile index fcafd7e..0eb9528 100644 --- a/Dockerfile.ct.Dockerfile +++ b/Dockerfile.ct.Dockerfile @@ -81,6 +81,9 @@ ENV CRAY_VAULT_AUTH_PATH="auth/token/create" ENV CRAY_VAULT_ROLE_FILE="/configs/namespace" ENV CRAY_VAULT_JWT_FILE="/configs/token" +ENV PCS_JAWS_MONITOR="true" +ENV PCS_JAWS_MONITOR_INTERVAL="20" + #nobody 65534:65534 USER 65534:65534 diff --git a/Dockerfile.pprof b/Dockerfile.pprof index af08c64..95ea916 100644 --- a/Dockerfile.pprof +++ b/Dockerfile.pprof @@ -80,6 +80,9 @@ ENV CRAY_VAULT_AUTH_PATH="auth/token/create" ENV CRAY_VAULT_ROLE_FILE="/configs/namespace" ENV CRAY_VAULT_JWT_FILE="/configs/token" +ENV PCS_JAWS_MONITOR="true" +ENV PCS_JAWS_MONITOR_INTERVAL="20" + #nobody 65534:65534 USER 65534:65534 diff --git a/Dockerfile.test.unit.Dockerfile b/Dockerfile.test.unit.Dockerfile index 362d685..8ee2c9c 100644 --- a/Dockerfile.test.unit.Dockerfile +++ b/Dockerfile.test.unit.Dockerfile @@ -48,6 +48,9 @@ ENV CRAY_VAULT_AUTH_PATH="auth/token/create" ENV CRAY_VAULT_ROLE_FILE="/go/configs/namespace" ENV CRAY_VAULT_JWT_FILE="/go/configs/token" +ENV PCS_JAWS_MONITOR="true" +ENV PCS_JAWS_MONITOR_INTERVAL="20" + RUN go env -w GO111MODULE=auto COPY cmd $GOPATH/src/github.com/OpenCHAMI/power-control/v2/cmd diff --git a/cmd/power-control/service.go b/cmd/power-control/service.go index 433588d..f730a8a 100644 --- a/cmd/power-control/service.go +++ b/cmd/power-control/service.go @@ -477,6 +477,35 @@ func runPCS(pcs *pcsConfig, etcd *etcdConfig, postgres *storage.PostgresConfig) logger.Log, (time.Duration(pwrSampleInterval) * time.Second), statusTimeout, statusHttpRetries, maxIdleConns, maxIdleConnsPerHost) + // Start up Monitoring of JAWS PDU devices + jawsMonitor := false + envstr = os.Getenv("PCS_JAWS_MONITOR") + if envstr != "" { + yn, _ := strconv.ParseBool(envstr) + if yn == true { + logger.Log.Infof("Monitoring JAWS devices.") + jawsMonitor = true + } + } else { + logger.Log.Infof("JAWS Monitoring set to False") + } + + if jawsMonitor { + jawsMonitorInterval := 30 + envstr = os.Getenv("PCS_JAWS_MONITOR_INTERVAL") + if envstr != "" { + jmi, err := strconv.Atoi(envstr) + if err != nil { + logger.Log.Errorf("Invalid value of PCS_JAWS_MONITOR_INTERVAL, defaulting to %d", + jawsMonitorInterval) + } else { + logger.Log.Infof("Using PCS_JAWS_MONITOR_INTERVAL: %v", jmi) + jawsMonitorInterval = jmi + } + } + go domain.JawsMonitor(jawsMonitorInterval) + } + domain.StartRecordsReaper() /////////////////////////////// diff --git a/internal/domain/jaws.go b/internal/domain/jaws.go new file mode 100644 index 0000000..016717d --- /dev/null +++ b/internal/domain/jaws.go @@ -0,0 +1,204 @@ +// MIT License +// +// (C) Copyright [2025] Hewlett Packard Enterprise Development LP +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. + +package domain + +import ( + "context" + "crypto/tls" + "encoding/json" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" + "github.com/sirupsen/logrus" + + "github.com/OpenCHAMI/power-control/v2/internal/logger" + pcsmodel "github.com/OpenCHAMI/power-control/v2/internal/model" +) + +type JawsEndpointStatus struct { + Id string `json:"id"` + Name string `json:"name"` + Active_power int `json:"active_power"` + Active_power_status string `json:"active_power_status"` + Apparent_power int `json:"apparent_power"` + Branch_id string `json:"branch_id"` + Control_state string `json:"control_state"` + Current float32 `json:"current"` + Current_capacity int `json:"current_capacity"` + Current_status string `json:"current_status"` + Current_utilized float32 `json:"current_utilized"` + Energy int `json:"energy"` + Ocp_id string `json:"ocp_id"` + Phase_id string `json:"phase_id"` + Power_capacity int `json:"power_capacity"` + Power_factor_status string `json:"power_factor_status"` + Socket_adapter string `json:"socket_adapter"` + Socket_type string `json:"socket_type"` + State string `json:"state"` + Status string `json:"status"` + Voltage float32 `json:"voltage"` +} + +func jawsSuites() []uint16 { + suites := []uint16{tls.TLS_RSA_WITH_AES_128_GCM_SHA256, tls.TLS_RSA_WITH_AES_256_GCM_SHA384, tls.TLS_RSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA} + for _, c := range tls.CipherSuites() { + suites = append(suites, c.ID) + } + return suites +} + +// Load JAWS outlets and store +func JawsLoad(xname string, FQDN string, authUser string, authPass string) { + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + CipherSuites: jawsSuites(), + } + + timeout := 20 + transport := &http.Transport{ + // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: tlsConfig, + } + client := retryablehttp.NewClient() + client.HTTPClient.Transport = transport + + var req *retryablehttp.Request + + // jaws/monitor/outlets + jurl, _ := url.Parse("https://" + FQDN + "/jaws/monitor/outlets") + req, err := retryablehttp.NewRequest("GET", jurl.String(), nil) + if err != nil { + logger.Log.Error(err) + return + } + + reqContext, reqCtxCancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout)) + req = req.WithContext(reqContext) + + req.Header.Add("Content-Type", "application/json") + req.Header.Add("cache-control", "no-cache") + + if !(authUser == "" && authPass == "") { + req.SetBasicAuth(authUser, authPass) + } + + resp, err := client.Do(req) + defer drainAndCloseBodyWithCtxCancel(resp, reqCtxCancel) + if err != nil { + logger.Log.Error(err) + return + } + + body, err := io.ReadAll(resp.Body) + var eps []JawsEndpointStatus + if err != nil { + logger.Log.Error(err) + return + } + err = json.Unmarshal(body, &eps) + if err != nil { + logger.Log.Error(err) + return + } + + // Store power state for each outlet + for _, jep2 := range eps { + epxname := jaws2xname(xname, jep2.Id) + + powerState := pcsmodel.PowerStateFilter_Undefined + if strings.EqualFold(jep2.State, "on") { + powerState = pcsmodel.PowerStateFilter_On + } else if strings.EqualFold(jep2.State, "off") { + powerState = pcsmodel.PowerStateFilter_Off + } + + updateHWState(epxname, powerState, pcsmodel.ManagementStateFilter_available, "") + } + return +} + +// Convert JAWS outlet name to an xname +// example: x3000m0p0v17 +func jaws2xname(xname string, id string) string { + nxname := xname + nxname = nxname + "p" + strconv.Itoa(int(id[0])-int('A')) + nxname = nxname + "v" + id[2:] + return nxname +} + +// JAWS loop to monitor PDUs +func JawsMonitor(looptime int) { + logger.Log.Info("In JAWS Monitor") + if looptime == 0 { + looptime = 20 + } + // Loop forever + for { + xnameList := []string{} + // Find PDUs + compMap, err := (*hsmHandle).FillHSMData([]string{"all"}) + if err != nil { + logger.Log.Error("JAWS FillHMSDATA ERROR: ", err) + } else { + for _, xname := range compMap { + if xname.BaseData.Type == "CabinetPDUController" { + if xname.PowerStatusURI == "/jaws" { + xnameList = append(xnameList, xname.BaseData.ID) + } + } + } + } + for _, xname := range xnameList { + logger.Log.Info("JAWS Load: " + xname) + var user, pw string + if GLOB.VaultEnabled { + user, pw, err = (*GLOB.CS).GetControllerCredentials(xname) + if err != nil { + logger.Log.WithFields(logrus.Fields{"ERROR": err}).Error("Unable to get credentials for " + xname) + } + } + JawsLoad(xname, xname, user, pw) + } + // Wait for a bit + time.Sleep(time.Duration(looptime) * time.Second) + } +} + +func drainAndCloseBodyWithCtxCancel(resp *http.Response, ctxCancel context.CancelFunc) { + // Must always drain and close response bodies + if resp != nil && resp.Body != nil { + _, _ = io.Copy(io.Discard, resp.Body) // ok even if already drained + resp.Body.Close() + } + // Call context cancel function, if supplied. This must be done after + // draining and closing the response body + if ctxCancel != nil { + ctxCancel() + } +} diff --git a/internal/domain/power-status.go b/internal/domain/power-status.go index 4f79b43..6482f56 100644 --- a/internal/domain/power-status.go +++ b/internal/domain/power-status.go @@ -536,6 +536,10 @@ func getHWStatesFromHW() error { glogger.Warnf("%s: Missing FQDN or power status URI for %s", fname, k) taskList[taskIX].Ignore = true taskIX++ + // PDU JAWS Device, handled in own monitoring loop + } else if strings.Contains(v.HSMData.PowerStatusURI, "/jaws") { + taskList[taskIX].Ignore = true + taskIX++ } else { url = "https://" + v.HSMData.RfFQDN + v.HSMData.PowerStatusURI taskList[taskIX].Request, _ = http.NewRequest(http.MethodGet, url, nil) diff --git a/internal/domain/transitions.go b/internal/domain/transitions.go index 46ed795..d46ecdd 100644 --- a/internal/domain/transitions.go +++ b/internal/domain/transitions.go @@ -784,7 +784,12 @@ func doTransition(transitionID uuid.UUID) { comp.Task.Operation = powerActionOp trsTaskMap[trsTaskList[trsTaskIdx].GetID()] = comp trsTaskList[trsTaskIdx].CPolicy.Retry.Retries = 3 - trsTaskList[trsTaskIdx].Request, _ = http.NewRequest("POST", "https://"+comp.HSMData.RfFQDN+comp.HSMData.PowerActionURI, bytes.NewBuffer([]byte(payload))) + method := "POST" + // JAWS Update uses the PATCH method + if strings.Contains(comp.HSMData.PowerActionURI, "/jaws") { + method = "PATCH" + } + trsTaskList[trsTaskIdx].Request, _ = http.NewRequest(method, "https://"+comp.HSMData.RfFQDN+comp.HSMData.PowerActionURI, bytes.NewBuffer([]byte(payload))) trsTaskList[trsTaskIdx].Request.Header.Set("Content-Type", "application/json") trsTaskList[trsTaskIdx].Request.Header.Add("HMS-Service", GLOB.BaseTRSTask.ServiceName) // Vault enabled? @@ -1600,6 +1605,10 @@ func generateTransitionPayload(comp *TransitionComponent, action string) (string } else { body = fmt.Sprintf(`{"PowerState": "%s"}`, resetType) } + // If we have a JAWS PDU + if strings.Contains(comp.HSMData.PowerActionURI, "jaws") { + body = fmt.Sprintf(`{"control_action": "%s"}`, resetType) + } } else { body = fmt.Sprintf(`{"ResetType": "%s"}`, resetType) }