From 03ede4a7845c294af2fe395cfdf572164506a41f Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Tue, 26 Nov 2024 01:22:08 -0500 Subject: [PATCH] Add disk generation to inventory (#7162) This is the first PR related to fixing #7098 --- nexus-sled-agent-shared/src/inventory.rs | 1 + nexus/db-model/src/inventory.rs | 4 ++++ nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/schema_versions.rs | 3 ++- nexus/db-queries/src/db/datastore/inventory.rs | 9 +++++++++ nexus/db-queries/src/db/datastore/physical_disk.rs | 2 ++ nexus/inventory/src/builder.rs | 2 ++ nexus/inventory/src/examples.rs | 1 + nexus/reconfigurator/planning/src/system.rs | 2 ++ nexus/types/src/inventory.rs | 10 ++++++++++ openapi/sled-agent.json | 4 ++++ schema/crdb/dbinit.sql | 5 ++++- .../crdb/inv-omicron-physical-disks-generation/up1.sql | 3 +++ .../crdb/inv-omicron-physical-disks-generation/up2.sql | 2 ++ sled-agent/src/rack_setup/plan/service.rs | 1 + sled-agent/src/rack_setup/service.rs | 1 + sled-agent/src/sim/sled_agent.rs | 1 + sled-agent/src/sled_agent.rs | 1 + 18 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 schema/crdb/inv-omicron-physical-disks-generation/up1.sql create mode 100644 schema/crdb/inv-omicron-physical-disks-generation/up2.sql diff --git a/nexus-sled-agent-shared/src/inventory.rs b/nexus-sled-agent-shared/src/inventory.rs index ebc683a264..5fb2d55203 100644 --- a/nexus-sled-agent-shared/src/inventory.rs +++ b/nexus-sled-agent-shared/src/inventory.rs @@ -103,6 +103,7 @@ pub struct Inventory { pub disks: Vec, pub zpools: Vec, pub datasets: Vec, + pub omicron_physical_disks_generation: Generation, } /// Describes the role of the sled within the rack. diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index c2383eafe3..61b5cc7edd 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -809,6 +809,7 @@ pub struct InvSledAgent { pub usable_hardware_threads: SqlU32, pub usable_physical_ram: ByteCount, pub reservoir_size: ByteCount, + pub omicron_physical_disks_generation: Generation, } impl InvSledAgent { @@ -852,6 +853,9 @@ impl InvSledAgent { sled_agent.usable_physical_ram, ), reservoir_size: ByteCount::from(sled_agent.reservoir_size), + omicron_physical_disks_generation: Generation::from( + sled_agent.omicron_physical_disks_generation, + ), }) } } diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 218cad84da..dd0b5e6ea3 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1486,6 +1486,7 @@ table! { usable_hardware_threads -> Int8, usable_physical_ram -> Int8, reservoir_size -> Int8, + omicron_physical_disks_generation -> Int8, } } diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 70450a7776..03c0e2c4e3 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(114, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(115, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(115, "inv-omicron-physical-disks-generation"), KnownVersion::new(114, "crucible-ref-count-records"), KnownVersion::new(113, "add-tx-eq"), KnownVersion::new(112, "blueprint-dataset"), diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 97d4b6357c..a6e2a6cf2a 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -889,6 +889,10 @@ impl DataStore { sled_agent.reservoir_size, ) .into_sql::(), + nexus_db_model::Generation( + sled_agent.omicron_physical_disks_generation, + ) + .into_sql::(), )) .filter( baseboard_dsl::part_number @@ -914,6 +918,7 @@ impl DataStore { sa_dsl::usable_hardware_threads, sa_dsl::usable_physical_ram, sa_dsl::reservoir_size, + sa_dsl::omicron_physical_disks_generation, )) .execute_async(&conn) .await?; @@ -933,6 +938,7 @@ impl DataStore { _usable_hardware_threads, _usable_physical_ram, _reservoir_size, + _omicron_physical_disks_generation, ) = sa_dsl::inv_sled_agent::all_columns(); } @@ -2377,6 +2383,9 @@ impl DataStore { .get(sled_id.as_untyped_uuid()) .map(|datasets| datasets.to_vec()) .unwrap_or_default(), + omicron_physical_disks_generation: s + .omicron_physical_disks_generation + .into(), }; sled_agents.insert(sled_id, sled_agent); } diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs index 52c93239aa..989aae0267 100644 --- a/nexus/db-queries/src/db/datastore/physical_disk.rs +++ b/nexus/db-queries/src/db/datastore/physical_disk.rs @@ -702,6 +702,8 @@ mod test { disks, zpools: vec![], datasets: vec![], + omicron_physical_disks_generation: + omicron_common::api::external::Generation::new(), }, ) .unwrap(); diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs index 7142b4f46c..e54b0c10ae 100644 --- a/nexus/inventory/src/builder.rs +++ b/nexus/inventory/src/builder.rs @@ -543,6 +543,8 @@ impl CollectionBuilder { .into_iter() .map(|d| d.into()) .collect(), + omicron_physical_disks_generation: inventory + .omicron_physical_disks_generation, }; if let Some(previous) = self.sleds.get(&sled_id) { diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs index 8279465475..03006d5e6f 100644 --- a/nexus/inventory/src/examples.rs +++ b/nexus/inventory/src/examples.rs @@ -573,5 +573,6 @@ pub fn sled_agent( disks, zpools, datasets, + omicron_physical_disks_generation: Generation::new(), } } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 28b8bcf2ae..05e73ac424 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -682,6 +682,7 @@ impl Sled { }) .collect(), datasets: vec![], + omicron_physical_disks_generation: Generation::new(), } }; @@ -820,6 +821,7 @@ impl Sled { disks: vec![], zpools: vec![], datasets: vec![], + omicron_physical_disks_generation: Generation::new(), }; Sled { diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index b38e8f3077..95c4f80e05 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -25,6 +25,7 @@ use nexus_sled_agent_shared::inventory::OmicronZoneConfig; use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_sled_agent_shared::inventory::SledRole; use omicron_common::api::external::ByteCount; +use omicron_common::api::external::Generation; pub use omicron_common::api::internal::shared::NetworkInterface; pub use omicron_common::api::internal::shared::NetworkInterfaceKind; pub use omicron_common::api::internal::shared::SourceNatConfig; @@ -520,4 +521,13 @@ pub struct SledAgent { pub disks: Vec, pub zpools: Vec, pub datasets: Vec, + /// As part of reconfigurator planning we need to know the control plane + /// disks configuration that the sled-agent has seen last. Specifically, + /// this allows the planner to know if a disk expungement has been seen by + /// the sled-agent, so that the planner can decommission the expunged disk. + /// + /// This field corresponds to the `generation` field in + /// `OmicronPhysicalDisksConfig` that is stored in the blueprint and sent to + /// the sled-agent via the executor over the internal API. + pub omicron_physical_disks_generation: Generation, } diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index be20ea5549..e8884cc86a 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -3942,6 +3942,9 @@ "$ref": "#/components/schemas/InventoryDisk" } }, + "omicron_physical_disks_generation": { + "$ref": "#/components/schemas/Generation" + }, "omicron_zones": { "$ref": "#/components/schemas/OmicronZonesConfig" }, @@ -3976,6 +3979,7 @@ "baseboard", "datasets", "disks", + "omicron_physical_disks_generation", "omicron_zones", "reservoir_size", "sled_agent_address", diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index f689c7e9f7..f95efa1a2f 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3353,6 +3353,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_agent ( usable_physical_ram INT8 NOT NULL, reservoir_size INT8 CHECK (reservoir_size < usable_physical_ram) NOT NULL, + -- The last generation of OmicronPhysicalDisksConfig seen by the sled-agent + omicron_physical_disks_generation INT8 NOT NULL, + PRIMARY KEY (inv_collection_id, sled_id) ); @@ -4684,7 +4687,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '114.0.0', NULL) + (TRUE, NOW(), NOW(), '115.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/inv-omicron-physical-disks-generation/up1.sql b/schema/crdb/inv-omicron-physical-disks-generation/up1.sql new file mode 100644 index 0000000000..e99a171ab6 --- /dev/null +++ b/schema/crdb/inv-omicron-physical-disks-generation/up1.sql @@ -0,0 +1,3 @@ +ALTER TABLE omicron.public.inv_sled_agent + ADD COLUMN IF NOT EXISTS omicron_physical_disks_generation INT8 + NOT NULL DEFAULT 1; diff --git a/schema/crdb/inv-omicron-physical-disks-generation/up2.sql b/schema/crdb/inv-omicron-physical-disks-generation/up2.sql new file mode 100644 index 0000000000..b0ba1ec938 --- /dev/null +++ b/schema/crdb/inv-omicron-physical-disks-generation/up2.sql @@ -0,0 +1,2 @@ +ALTER TABLE omicron.public.inv_sled_agent + ALTER COLUMN omicron_physical_disks_generation DROP DEFAULT; diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index e7dd5a2b37..2b5e35f036 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -1557,6 +1557,7 @@ mod tests { disks, zpools: vec![], datasets: vec![], + omicron_physical_disks_generation: Generation::new(), }, is_scrimlet, )]; diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index 05283a835e..e8beb5ea23 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -1805,6 +1805,7 @@ mod test { .collect(), zpools: vec![], datasets: vec![], + omicron_physical_disks_generation: Generation::new(), }, true, ) diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 7b797eb45a..1f099fc036 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -916,6 +916,7 @@ impl SledAgent { .collect::>() }) .unwrap_or_else(|_| vec![]), + omicron_physical_disks_generation: Generation::new(), }) } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 2a14588401..80dbe72ea3 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -1361,6 +1361,7 @@ impl SledAgent { disks, zpools, datasets, + omicron_physical_disks_generation: *all_disks.generation(), }) }