Skip to content

Commit

Permalink
[NodeKiller] Add randomization on node killing
Browse files Browse the repository at this point in the history
  • Loading branch information
jprzychodzen committed Mar 5, 2020
1 parent e658b72 commit 9f57ad5
Showing 1 changed file with 11 additions and 4 deletions.
15 changes: 11 additions & 4 deletions clusterloader2/pkg/chaos/nodes.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ func (k *NodeKiller) pickNodes() ([]v1.Node, error) {
for i := range prometheusPods {
if prometheusPods[i].Spec.NodeName != "" {
nodesHasPrometheusPod.Insert(prometheusPods[i].Spec.NodeName)
klog.Infof("%s: Node %s removed from killing. Runs pod %s", k, prometheusPods[i].Spec.NodeName, prometheusPods[i].Name)
}
}

Expand All @@ -98,11 +99,17 @@ func (k *NodeKiller) pickNodes() ([]v1.Node, error) {
rand.Shuffle(len(nodes), func(i, j int) {
nodes[i], nodes[j] = nodes[j], nodes[i]
})
numNodes := int(k.config.FailureRate * float64(len(nodes)))
if len(nodes) > numNodes {
return nodes[:numNodes], nil
expectedNodesToFail := k.config.FailureRate * float64(len(nodes))
nodesToFail := nodes[:0]
klog.Infof("%s: %d nodes available, estimating to select %f nodes", k, len(nodes), expectedNodesToFail)
for _, node := range nodes {
if rand.Float64() < k.config.FailureRate {
klog.Infof("%s: Node %q schedule for failure", k, node.Name)
nodesToFail = append(nodesToFail, node)
}
}
return nodes, nil
klog.Infof("%s: Total %d nodes scheduled for failure", k, len(nodesToFail))
return nodesToFail, nil
}

func (k *NodeKiller) kill(nodes []v1.Node) {
Expand Down

0 comments on commit 9f57ad5

Please sign in to comment.