Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion charts/topograph/values.slinky.block-example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ global:
matchLabels:
app.kubernetes.io/component: compute
plugin: topology/block
block_sizes: 4
blockSizes: [4]
topologyConfigPath: topology.conf
topologyConfigmapName: slurm-config
reportingMode: staticNodes
Expand Down
6 changes: 3 additions & 3 deletions docs/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,13 @@ Topograph exposes three endpoints for interacting with the service. Below are th
- **slurm parameters**:
- **topologyConfigPath**: (optional) A string specifying the file path for the topology configuration. If omitted, the topology config content is returned in the HTTP response.
- **plugin**: (optional) A string specifying topology plugin: `topology/tree` (default) or `topology/block`.
- **block_sizes**: (optional) A string specifying block size for `topology/block` plugin.
- **blockSizes**: (optional) A string specifying block size for `topology/block` plugin.
- **reconfigure**: (optional) If `true`, invoke `scontrol reconfigure` after topology config is generated. Default `false`
- **slinky parameters**:
- **namespace**: A string specifying namespace where SLURM cluster is running.
- **podSelector**: A standard Kubernetes label selector for pods running SLURM nodes.
- **plugin**: (optional) A string specifying topology plugin: `topology/tree` (default) or `topology/block`.
- **block_sizes**: (optional) A string specifying block size for `topology/block` plugin.
- **blockSizes**: (optional) A string specifying block size for `topology/block` plugin.
- **topologyConfigPath**: A string specifying the key for the topology config in the ConfigMap.
- **topologyConfigmapName**: A string specifying the name of the ConfigMap containing the topology config.
- **nodes**: (optional) An array of regions mapping instance IDs to node names.
Expand All @@ -111,7 +111,7 @@ Topograph exposes three endpoints for interacting with the service. Below are th
"name": "slurm",
"params": {
"plugin": "topology/block",
"block_sizes": "30,120"
"blockSizes": [30,120]
}
},
"nodes": [
Expand Down
14 changes: 7 additions & 7 deletions docs/engines/slinky.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@ global:
provider: aws
engine: slinky
engineParams:
namespace: ns-slinky # Namespace where Slinky is running
podSelector: # Label selector for pods running SLURM nodes
namespace: ns-slinky # Namespace where Slinky is running
podSelector: # Label selector for pods running SLURM nodes
matchLabels:
app.kubernetes.io/component: compute
plugin: topology/block # Name of the topology plugin
block_sizes: 4 # (Optional) Block size for the block topology plugin
topologyConfigmapName: slurm-config # Name of the ConfigMap containing the topology config
topologyConfigPath: topology.conf # Key in the ConfigMap for the topology config
plugin: topology/block # Name of the topology plugin
blockSizes: [4] # (Optional) Block size for the block topology plugin
topologyConfigmapName: slurm-config # Name of the ConfigMap containing the topology config
topologyConfigPath: topology.conf # Key in the ConfigMap for the topology config
```

### Per-partition topologies
Expand Down Expand Up @@ -160,7 +160,7 @@ curl -X POST -H "Content-Type: application/json" \
"topologyConfigPath": "topology.conf",
"topologyConfigmapName": "slurm-config",
"plugin": "topology/block",
"block_sizes": "8,16,32"
"blockSizes": [8,16,32]
}
}
}' \
Expand Down
2 changes: 1 addition & 1 deletion docs/engines/slurm.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ id=$(curl -s -X POST -H "Content-Type: application/json" -d '{"provider":{"param

And if you want to view the block topology (with specified block sizes), use the command:
```bash
id=$(curl -s -X POST -H "Content-Type: application/json" -d '{"provider":{"params":{"modelFileName":"/usr/local/bin/tests/models/<cluster-model>.yaml"}},"engine":{"params":{"plugin":"topology/block", "block_sizes": "4,8"}}}' http://localhost:49021/v1/generate)
id=$(curl -s -X POST -H "Content-Type: application/json" -d '{"provider":{"params":{"modelFileName":"/usr/local/bin/tests/models/<cluster-model>.yaml"}},"engine":{"params":{"plugin":"topology/block", "blockSizes": [4,8]}}}' http://localhost:49021/v1/generate)
```

You can query the results of either topology request with:
Expand Down
2 changes: 1 addition & 1 deletion docs/get-started/quickstart-k8s.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ Engine-specific install flags point the `slinky` engine at the Slinky deployment
--set global.engineParams.topologyConfigPath=topology.conf
```

The full engine-parameter shape (`podSelector`, `plugin`, `block_sizes`, per-partition topologies, …) is documented in the [chart README](https://github.com/NVIDIA/topograph/blob/main/charts/topograph/README.md). Example values files for common Slinky scenarios ship in the chart directory:
The full engine-parameter shape (`podSelector`, `plugin`, `blockSizes`, per-partition topologies, …) is documented in the [chart README](https://github.com/NVIDIA/topograph/blob/main/charts/topograph/README.md). Example values files for common Slinky scenarios ship in the chart directory:

- [`values.slinky.tree-example.yaml`](https://github.com/NVIDIA/topograph/blob/main/charts/topograph/values.slinky.tree-example.yaml) — tree topology
- [`values.slinky.block-example.yaml`](https://github.com/NVIDIA/topograph/blob/main/charts/topograph/values.slinky.block-example.yaml) — block topology
Expand Down
12 changes: 11 additions & 1 deletion pkg/engines/slinky/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"fmt"
"maps"
"net/http"
"strconv"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -258,7 +260,7 @@ func (eng *SlinkyEngine) generateConfigMapAnnotations() map[string]string {
annotations[topology.KeyConfigMapPlugin] = eng.params.Plugin
}
if len(eng.params.BlockSizes) != 0 {
annotations[topology.KeyConfigMapBlockSizes] = eng.params.BlockSizes
annotations[topology.KeyConfigMapBlockSizes] = intToStr(eng.params.BlockSizes)
}

return annotations
Expand Down Expand Up @@ -545,3 +547,11 @@ func (eng *SlinkyEngine) updateNodeAnnotation(ctx context.Context, node *corev1.

return nil
}

func intToStr(input []int) string {
strs := make([]string, len(input))
for i, n := range input {
strs[i] = strconv.Itoa(n)
}
return strings.Join(strs, ",")
}
14 changes: 7 additions & 7 deletions pkg/engines/slinky/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,14 +143,14 @@ func TestGetParameters(t *testing.T) {
topology.KeyPodSelector: podSelector,
topology.KeyNodeSelector: nodeSelector,
topology.KeyPlugin: topology.TopologyBlock,
topology.KeyBlockSizes: "16",
topology.KeyBlockSizes: []int{16},
topology.KeyTopoConfigPath: "path",
topology.KeyTopoConfigmapName: "name",
},
ret: &Params{
BaseParams: slurm.BaseParams{
Plugin: topology.TopologyBlock,
BlockSizes: "16",
BlockSizes: []int{16},
},
Namespace: "namespace",
PodSelector: labelSelector,
Expand Down Expand Up @@ -343,7 +343,7 @@ func TestConfigMapAnnotationsAndMetadata(t *testing.T) {
name: "with block sizes only",
params: &Params{
BaseParams: slurm.BaseParams{
BlockSizes: "8,16,32",
BlockSizes: []int{8, 16, 32},
},
Namespace: "test-namespace",
PodSelector: labelSelector,
Expand All @@ -357,7 +357,7 @@ func TestConfigMapAnnotationsAndMetadata(t *testing.T) {
params: &Params{
BaseParams: slurm.BaseParams{
Plugin: topology.TopologyBlock,
BlockSizes: "8,16,32",
BlockSizes: []int{8, 16, 32},
},
Namespace: "test-namespace",
PodSelector: labelSelector,
Expand Down Expand Up @@ -388,19 +388,19 @@ func TestConfigMapAnnotationsAndMetadata(t *testing.T) {
require.NotContains(t, annotations, topology.KeyConfigMapPlugin)
}
if tc.wantBlock {
requireAnnotation(t, annotations, topology.KeyConfigMapBlockSizes, tc.params.BlockSizes)
requireAnnotation(t, annotations, topology.KeyConfigMapBlockSizes, intToStr(tc.params.BlockSizes))
} else {
require.NotContains(t, annotations, topology.KeyConfigMapBlockSizes)
}

// Metadata logic (simulate GenerateOutput)
tree := &topology.Vertex{Name: "root"}
setMetadata(tree, tc.params.Plugin, tc.params.BlockSizes)
setMetadata(tree, tc.params.Plugin, intToStr(tc.params.BlockSizes))
if tc.wantPlugin {
require.Equal(t, tc.params.Plugin, tree.Metadata[topology.KeyPlugin])
}
if tc.wantBlock {
require.Equal(t, tc.params.BlockSizes, tree.Metadata[topology.KeyBlockSizes])
require.Equal(t, intToStr(tc.params.BlockSizes), tree.Metadata[topology.KeyBlockSizes])
}
})
}
Expand Down
71 changes: 3 additions & 68 deletions pkg/engines/slurm/slurm.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"fmt"
"net/http"
"regexp"
"strconv"
"strings"

"k8s.io/klog/v2"
Expand All @@ -34,7 +33,6 @@ import (
"github.com/NVIDIA/topograph/internal/files"
"github.com/NVIDIA/topograph/internal/httperr"
"github.com/NVIDIA/topograph/pkg/engines"
"github.com/NVIDIA/topograph/pkg/metrics"
"github.com/NVIDIA/topograph/pkg/topology"
"github.com/NVIDIA/topograph/pkg/translate"
)
Expand All @@ -51,10 +49,8 @@ const NAME = "slurm"
type SlurmEngine struct{}

type BaseParams struct {
Plugin string `mapstructure:"plugin"`
BlockSizes string `mapstructure:"block_sizes"`
FakeNodesEnabled bool `mapstructure:"fakeNodesEnabled"`
FakeNodePool string `mapstructure:"fake_node_pool"`
Plugin string `mapstructure:"plugin"`
BlockSizes []int `mapstructure:"blockSizes"`
}

type Topology struct {
Expand Down Expand Up @@ -184,34 +180,6 @@ func GetNodeList(ctx context.Context) ([]string, error) {
return nodes, nil
}

func GetFakeNodes(ctx context.Context) (string, error) {
args := []string{"show", "partition", "fake"}
stdout, err := exec.Exec(ctx, "scontrol", args, nil)
if err != nil {
return "", err
}
out := stdout.String()
klog.V(4).Infof("stdout: %s", out)

return parseFakeNodes(out)
}

func parseFakeNodes(data string) (string, error) {
prefix := "Nodes="
scanner := bufio.NewScanner(strings.NewReader(data))
for scanner.Scan() {
if line := strings.TrimSpace(scanner.Text()); strings.HasPrefix(line, prefix) {
return line[len(prefix):], nil
}
}

if err := scanner.Err(); err != nil {
return "", fmt.Errorf("failed to scan fake nodes partition: %v", err)
}

return "", fmt.Errorf("fake partition has no nodes")
}

func getPartitionNodes(ctx context.Context, partition string, _ []any) (string, error) {
args := []string{"show", "partition", partition}
stdout, err := exec.Exec(ctx, "scontrol", args, nil)
Expand Down Expand Up @@ -307,22 +275,7 @@ func GenerateOutputParams(ctx context.Context, root *topology.Vertex, params *Pa
func GetTranslateConfig(ctx context.Context, params *BaseParams, topologies map[string]*Topology, f *TopologyNodeFinder) (*translate.Config, error) {
cfg := &translate.Config{
Plugin: params.Plugin,
BlockSizes: getBlockSizes(params.BlockSizes),
}

// set fake nodes
if params.Plugin == topology.TopologyBlock && params.FakeNodesEnabled {
var fakeNodes string
var err error
if len(params.FakeNodePool) > 0 {
fakeNodes = params.FakeNodePool
} else {
fakeNodes, err = GetFakeNodes(ctx)
if err != nil {
return nil, err
}
}
cfg.FakeNodePool = fakeNodes
BlockSizes: params.BlockSizes,
}

// set per-partition topologies
Expand Down Expand Up @@ -361,24 +314,6 @@ func getParams(params map[string]any) (*Params, error) {
return &p, err
}

func getBlockSizes(str string) []int {
if len(str) == 0 {
return nil
}
parts := strings.Split(str, ",")
blockSizes := make([]int, 0, len(parts))
for _, part := range parts {
sz, err := strconv.Atoi(part)
if err != nil {
metrics.AddValidationError("BlockSize parsing error")
klog.Warningf("Failed to parse blockSize %v: %v. Ignoring admin blockSizes.", part, err)
return nil
}
blockSizes = append(blockSizes, sz)
}
return blockSizes
}

func reconfigure(ctx context.Context) error {
stdout, err := exec.Exec(ctx, "scontrol", []string{"reconfigure"}, nil)
if err != nil {
Expand Down
Loading
Loading