From 2478032e61fd6643c33bbe0ad3384bea1c08a3a8 Mon Sep 17 00:00:00 2001 From: cool-mist Date: Sun, 22 Jun 2025 00:25:44 +0530 Subject: [PATCH] Add parser --- .gitignore | 1 + Cargo.lock | 1594 ++++++++++++++++++++++++ Cargo.toml | 10 + docs/design.markdown | 12 + sample/rss.xml | 2373 ++++++++++++++++++++++++++++++++++++ sample/rss2.xml | 2754 ++++++++++++++++++++++++++++++++++++++++++ src/error.rs | 22 + src/main.rs | 26 + src/parser.rs | 221 ++++ 9 files changed, 7013 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 docs/design.markdown create mode 100644 sample/rss.xml create mode 100644 sample/rss2.xml create mode 100644 src/error.rs create mode 100644 src/main.rs create mode 100644 src/parser.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..02446b4 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1594 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "cc" +version = "1.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "h2" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "http" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.173" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8cfeafaffdbc32176b64fb251369d52ea9f0a8fbc6f8759edffef7b525d64bb" + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", +] + +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "redox_syscall" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +dependencies = [ + "bitflags", +] + +[[package]] +name = "reqwest" +version = "0.12.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4307e30089d6fd6aff212f2da3a1f9e32f3223b1f010fb09b7c95f90f3ca1e8" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + +[[package]] +name = "trs" +version = "0.1.0" +dependencies = [ + "hyper", + "reqwest", + "tokio", + "xml-rs", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-registry" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "xml-rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62ce76d9b56901b19a74f19431b0d8b3bc7ca4ad685a746dfd78ca8f4fc6bda" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..9a08c71 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "trs" +version = "0.1.0" +edition = "2021" + +[dependencies] +hyper = "1.6.0" +reqwest = "0.12.20" +tokio = { version = "1.45.1", features = ["full"] } +xml-rs = "0.8.26" diff --git a/docs/design.markdown b/docs/design.markdown new file mode 100644 index 0000000..eeb4bdb --- /dev/null +++ b/docs/design.markdown @@ -0,0 +1,12 @@ +# Terminal RSs + + +Keep the goal simple. + +- just list the unread articles +- Have a way to mark an article as read/unread +https://www.rssboard.org/rss-specification + +https://brycev.com/rss.xml + + diff --git a/sample/rss.xml b/sample/rss.xml new file mode 100644 index 0000000..0524c4f --- /dev/null +++ b/sample/rss.xml @@ -0,0 +1,2373 @@ + + + Bryce Vandegrift's Website + https://brycev.com/ + Updates to Bryce Vandegrift's blog + en-us + + + + + + New Domain Name + https://brycev.com/blog/new-domain-name/ + Wed, 14 May 2025 00:00:00 +0000 + https://brycev.com/blog/new-domain-name/ + <p>It&rsquo;s about time that I got a new domain name for my website. My current/old +domain name, <a href="https://brycevandegrift.xyz">brycevandegrift.xyz</a>, is quite a +cumbersome and unorthodox domain name. It&rsquo;s not easy to remember and the +<code>.xyz</code> domain name can also be somewhat confusing. Not only that, but I&rsquo;ve had +less tech literate people tell me that they thought my domain wasn&rsquo;t actually +a real domain name. Add to the fact that many email services automatically +mark <code>.xyz</code> domain names as spam, and you can see why I want to change my +domain name.</p> +<p>I originally got a <code>.xyz</code> domain name because it was fairly cheap (only +<strong>99ยข</strong>!), however using my <em>full</em> name was probably not the right way to go. +My emails also got marked as spam or just filtered by some email servers.</p> +<h2 id="new-domain-name">New domain name</h2> +<p>I&rsquo;ve been eyeballing <strong><a href="https://brycev.com">brycev.com</a></strong> for quite some time and +I finally pulled the trigger and bought it. <code>.com</code> domains are usually the +most memorable and trusted domain names out there so I decided to go with +that. Right now <code>brycev.com</code> just redirects to <code>brycevandegrift.xyz</code>, but over +the next few weeks I will be slowly replacing instances of my old domain name +with my new domain name. I will probably also keep <code>brycevandegrift.xyz</code> +indefinitely and have it redirect to <code>brycev.com</code>.</p> +<p>I&rsquo;ll slowly migrate my website, email, and XMPP server over to using my new +domain name. I&rsquo;ll also need to issue a new GPG/PGP key for my new email +address. So keep an eye out and I&rsquo;ll post any necessary updates.</p> + + + + + Blogging on Paper + https://brycev.com/blog/blogging-on-paper/ + Mon, 31 Mar 2025 00:00:00 +0000 + https://brycev.com/blog/blogging-on-paper/ + <p>I&rsquo;ve been wanting to post more articles on my blog, however I often +find myself away from my computer or I often have more important +tasks to take care of on my computer. Either way, I can&rsquo;t write +as many articles for my blog as I would like to on my computer.</p> +<p>It wasn&rsquo;t until I read a blog post by Ray Patrick (who +has sadly taken down his blog) titled +<a href="https://web.archive.org/web/20240229190333if_/https://raypatrick.xyz/blog/2024/01/24/in-praise-of-the-manual-typewriter/">&ldquo;In Praise of the Manual Typewriter&rdquo;</a> +that I formed an idea. I could type out the articles +that I want to compose on a typewriter. This would allow me to +formulate any articles that I would like to later post to my blog. +I could also handwrite any articles if needed.</p> +<p>I&rsquo;ve been in possession of a Royal Mercury typewriter +for quite some time now and I have been looking to get some use +out of it. It&rsquo;s a somewhat small, portable typewriter that is easy +to carry around. This means that I can take it with me almost anywhere +that I go. And guess what? I don&rsquo;t have to constantly keep it charged +like a laptop&hellip;although it does need paper and ink which could be +a slight disadvantage.</p> +<p>I also realized that I could use OCR (Optical Character +Recognition) programs like <a href="https://tesseract-ocr.github.io/">Tesseract</a> +in order to easily convert +anything that I type or write while I&rsquo;m away from my computer into +text to put on my blog. I might have to do some manual editing and +correcting once it&rsquo;s converted into plaintext, but I think that +the extra effort would be well worth it.</p> +<p>This method of writing articles might very much help me +to write more in general, not just for my blog. Keep an eye out +though, you might see me putting more on my blog/website within the +upcoming months.</p> +<hr> + + +<figure ><a href="https://brycev.com/p/paper-blog.webp"><img src="https://brycev.com/p/paper-blog.webp" alt="This post written on paper"></a></figure> + + + + + + OpenBSD Server: smtpd + https://brycev.com/blog/openbsd-smtpd/ + Fri, 07 Mar 2025 00:00:00 +0000 + https://brycev.com/blog/openbsd-smtpd/ + <p>At last, the final part to this series. You can catch up on parts 1 and 2 +<a href="https://brycev.com/blog/openbsd-server-install">here</a> and <a href="https://brycev.com/blog/openbsd-httpd">here</a> respectively. +Setting up an email server is no easy task, it takes a lot of configuration and +sometimes trial and error. This guide should be good enough for small email +servers that don&rsquo;t get too many emails. +So let&rsquo;s go ahead and finally setup an email server for our system.</p> +<h2 id="some-needed-packages">Some Needed Packages</h2> +<p>First, we need to install some extra packages for our email server to work the +way we want it to. We will need <code>rspamd</code> to filter out spam, <code>dovecot</code> in order +to use IMAP and or POP3 to view our emails, and we will need +<code>dovecot-pigeonhole</code> to help sort our emails:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas pkg_add rspamd-- dovecot-- dovecot-pigeonhole-- opensmtpd-filter-rspamd +</span></span></code></pre></div><h2 id="dns-records">DNS Records</h2> +<p>In order for our email server to properly work and also to not get filtered or +marked as spam, we will need to add some DNS records.</p> +<h3 id="mx-record">MX Record</h3> +<p>A MX record is required to tell other SMTP servers where to deliver emails +sent to our address. For our MX record we can have our emails redirected to +<code>mail.example.com</code><sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>. When creating an MX record, this is what it should look +like:</p> +<pre tabindex="0"><code>Record Type Host Value MX Priority TTL + V V V V V + MX @ mail.example.com 0 3600 +</code></pre><p>Our record type should (obviously) be MX, our host should be empty (which is +sometimes represented as an <code>@</code> symbol), our value is where our mail should go +which is <code>mail.example.com</code>, our MX priority should just be 0 for now, and our +TTL is set to 3600 seconds (or 1 hour).</p> +<p>For reference, this is how my MX record is setup for <code>brycevandegrift.xyz</code>:</p> + + +<figure ><a href="https://brycev.com/p/mx-record.webp"><img src="https://brycev.com/p/mx-record.webp" alt="My personal MX record"></a></figure> + +<h3 id="spf-record">SPF Record</h3> +<p>Our SPF record is vital in order to detect spam. It defines what servers can +send emails from a domain. In our case, only <code>mail.example.com</code> will be sending +emails on our behalf, so we can set our SPF record accordingly:</p> +<pre tabindex="0"><code>Record Type Host Value TTL + V V V V + TXT @ v=spf1 mx all 3600 +</code></pre><p>Our SPF record is just a normal TXT record that has the string &ldquo;v=spf1 mx all&rdquo; +in it which allows emails to be sent in reference to our MX record.</p> +<h3 id="dkim-record">DKIM Record</h3> +<p>We need a DKIM record in order to cryptographically sign and verify the +emails we send. Without DKIM your emails will <strong>NOT</strong> be accepted by most email +servers and will be marked as spam. In order to use DKIM we will need to +generate our private and public keys on our system. We can generate our keys and +give them the correct permissions like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span><span style="color:#75715e"># First enter root session</span> +</span></span><span style="display:flex;"><span>su +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span>umask <span style="color:#ae81ff">077</span> +</span></span><span style="display:flex;"><span>install -d -o root -g wheel -m <span style="color:#ae81ff">755</span> /etc/mail/dkim +</span></span><span style="display:flex;"><span>install -d -o root -g _rspamd -m <span style="color:#ae81ff">775</span> /etc/mail/dkim/private +</span></span><span style="display:flex;"><span>openssl genrsa -out /etc/mail/dkim/private/example.com.key <span style="color:#ae81ff">2048</span> +</span></span><span style="display:flex;"><span>openssl rsa -in /etc/mail/dkim/private/example.com.key -pubout -out /etc/mail/dkim/example.com.pub +</span></span><span style="display:flex;"><span>chgrp _rspamd /etc/mail/dkim/private/example.com.key /etc/mail/dkim/private/ +</span></span><span style="display:flex;"><span>chmod <span style="color:#ae81ff">440</span> /etc/mail/dkim/private/example.com.key +</span></span><span style="display:flex;"><span>chmod <span style="color:#ae81ff">775</span> /etc/mail/dkim/private/ +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Exit root session</span> +</span></span><span style="display:flex;"><span>exit +</span></span></code></pre></div><p>Now that we successfully generated our private and public DKIM keys, we need to +put our public DKIM key into our DKIM record. We can get our DKIM public key as +a single line by running this <code>awk</code> command:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas awk <span style="color:#e6db74">&#39;/PUBLIC/ { $0=&#34;&#34; } { printf (&#34;%s&#34;,$0) } END { print }&#39;</span> /etc/mail/dkim/example.com.pub +</span></span></code></pre></div><p>We can now copy that string and put it into our DKIM record.</p> +<pre tabindex="0"><code>Record Type Host Value TTL + V V V V + TXT dkim._domainkey v=DKIM1;k=rsa;p=&lt;YOUR DKIM KEY&gt; 3600 +</code></pre><p>Just replace <code>&lt;YOUR DKIM KEY&gt;</code> with the string you just copied and your DKIM +record is good to go.</p> +<h3 id="dmarc-record">DMARC Record</h3> +<p>We now need to add a DMARC record to our DNS records. DMARC is just another way +of preventing email spoofing and spam by telling receiving email servers what to +do with potentially invalid emails. Here is a good default to use for you DMARC +record:</p> +<pre tabindex="0"><code>Record Type Host Value TTL + V V V V + TXT @ v=DMARC1;p=reject;rua=mailto:dmarc@example.com;sp=reject;aspf=r; 3600 +</code></pre><p>Wow, the value for this record is quite long, let&rsquo;s break down what it means. +<code>p=reject</code> and <code>sp=reject</code> just means to immediately discard any invalid emails, +<code>aspf=r</code> means that <em>any</em> email sent from <em>any</em> subdomain of example.com is +valid, and <code>rua=dmarc@example.com</code> tells where the mail server receives DMARC +reports (don&rsquo;t worry, you shouldn&rsquo;t care about DMARC reports, they are mostly +useless).</p> +<h3 id="ptr-record">PTR Record</h3> +<p>Finally, our last record! All you need to do is set a PTR record (also called a +Reverse DNS pointer) that points to +<code>mail.example.com</code>. That&rsquo;s it. This is yet <strong>another</strong> mechanism to prevent +spam. Please note that there are different ways to set a PTR record, most ways +involve finding a &ldquo;Reverse DNS&rdquo; option for your VPS/hosting provider and using +that.</p> +<h2 id="email-configuration">Email Configuration</h2> +<p>At last, we can now configure the setting for our email server. Our first order +of business is to get a valid certificate for our <code>mail.example.com</code> subdomain. +This process should be somewhat familiar if you read my last article. Let&rsquo;s +first append this to the end of our <code>/etc/acme-client.conf</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">domain</span> <span style="color:#66d9ef">mail.example.com</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">domain</span> <span style="color:#66d9ef">key</span> <span style="color:#e6db74">&#34;/etc/ssl/private/mail.example.com.key&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">domain</span> <span style="color:#66d9ef">full</span> <span style="color:#66d9ef">chain</span> <span style="color:#66d9ef">certificate</span> <span style="color:#e6db74">&#34;/etc/ssl/mail.example.com.fullchain.pem&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">sign</span> <span style="color:#66d9ef">with</span> <span style="color:#66d9ef">letsencrypt</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>Now we just need to add this to <code>/etc/httpd.conf</code> in order to get a certificate +for <em>any</em> subdomain of example.com:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">server</span> <span style="color:#e6db74">&#34;*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#960050;background-color:#1e0010">*</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">80</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">location</span> <span style="color:#e6db74">&#34;/.well-known/acme-challenge/*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">root</span> <span style="color:#e6db74">&#34;/acme&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">request</span> <span style="color:#66d9ef">strip</span> <span style="color:#ae81ff">2</span> +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">location</span> <span style="color:#960050;background-color:#1e0010">*</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">block</span> <span style="color:#66d9ef">return</span> <span style="color:#ae81ff">302</span> <span style="color:#e6db74">&#34;https://$HTTP_HOST$REQUEST_URI&#34;</span> +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>Now we just run this command to generate a certificate for <code>mail.example.com</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas acme-client -v mail.example.com +</span></span></code></pre></div><p>Finally, we shouldn&rsquo;t forget to automatically renew our certificates, so change +<code>/etc/weekly.local</code> as follows:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span><span style="color:#75715e"># Check for new certificates every week</span> +</span></span><span style="display:flex;"><span>acme-client example.com +</span></span><span style="display:flex;"><span>acme-client mail.example.com +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span>rcctl restart httpd smtpd dovecot +</span></span></code></pre></div><h3 id="spam">Spam</h3> +<p>The first program that we should configure is <code>rspamd</code>. Only a few things need +to be set in order for it to work properly. Go ahead and create a file at +<code>/etc/rspamd/local.d/dkim_signing.conf</code> and add this to it:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#75715e"># This should always be true +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">allow_username_mismatch</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">true</span><span style="color:#960050;background-color:#1e0010">;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Allow rspamd to sign with our DKIM key +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">domain</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">example.com</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">path</span> <span style="color:#f92672">=</span> <span style="color:#e6db74">&#34;/etc/mail/dkim/private/example.com.key&#34;</span><span style="color:#960050;background-color:#1e0010">;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">selector</span> <span style="color:#f92672">=</span> <span style="color:#e6db74">&#34;dkim&#34;</span><span style="color:#960050;background-color:#1e0010">;</span> +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>This is entirely optional, but if you want more performance you can enable +and start <code>redis</code> as a cache backend:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas rcctl enable redis +</span></span><span style="display:flex;"><span>doas rcctl start redis +</span></span></code></pre></div><p>Now enable and start <code>rspamd</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas rcctl enable rspamd +</span></span><span style="display:flex;"><span>doas rcctl start rspamd +</span></span></code></pre></div><h3 id="smtp">SMTP</h3> +<p>Now let&rsquo;s configure OpenSMTPD in order to send and receive emails. We will need +to create a config file at <code>/etc/mail/smtpd.conf</code> and configure it like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#75715e"># Set locations to certificates +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">pki</span> <span style="color:#66d9ef">example.com</span> <span style="color:#66d9ef">cert</span> <span style="color:#e6db74">&#34;/etc/ssl/mail.example.com.fullchain.pem&#34;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">pki</span> <span style="color:#66d9ef">example.com</span> <span style="color:#66d9ef">key</span> <span style="color:#e6db74">&#34;/etc/ssl/private/mail.example.com.key&#34;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">pki</span> <span style="color:#66d9ef">example.com</span> <span style="color:#66d9ef">dhe</span> <span style="color:#66d9ef">auto</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Delimiter for email addresses +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">smtp</span> <span style="color:#66d9ef">sub-addr-delim</span> <span style="color:#960050;background-color:#1e0010">&#39;</span><span style="color:#66d9ef">_</span><span style="color:#960050;background-color:#1e0010">&#39;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Sets rspamd as our filter +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">filter</span> <span style="color:#66d9ef">rspamd</span> <span style="color:#66d9ef">proc-exec</span> <span style="color:#e6db74">&#34;filter-rspamd&#34;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Sets ports to use for mail transfer +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#66d9ef">all</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">25</span> <span style="color:#66d9ef">tls</span> <span style="color:#66d9ef">pki</span> <span style="color:#e6db74">&#34;example.com&#34;</span> <span style="color:#66d9ef">filter</span> <span style="color:#e6db74">&#34;rspamd&#34;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#66d9ef">all</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">465</span> <span style="color:#66d9ef">smtps</span> <span style="color:#66d9ef">pki</span> <span style="color:#e6db74">&#34;example.com&#34;</span> <span style="color:#66d9ef">auth</span> <span style="color:#66d9ef">mask-src</span> <span style="color:#66d9ef">filter</span> <span style="color:#e6db74">&#34;rspamd&#34;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#66d9ef">all</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">587</span> <span style="color:#66d9ef">tls-require</span> <span style="color:#66d9ef">pki</span> <span style="color:#e6db74">&#34;example.com&#34;</span> <span style="color:#66d9ef">auth</span> <span style="color:#66d9ef">mask-src</span> <span style="color:#66d9ef">filter</span> <span style="color:#e6db74">&#34;rspamd&#34;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Load mail aliases +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">table</span> <span style="color:#66d9ef">aliases</span> <span style="color:#66d9ef">file</span><span style="color:#960050;background-color:#1e0010">:</span>/etc/mail/aliases +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Actions for mail delievery +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">action</span> <span style="color:#e6db74">&#34;local&#34;</span> <span style="color:#66d9ef">lmtp</span> <span style="color:#e6db74">&#34;/var/dovecot/lmtp&#34;</span> <span style="color:#66d9ef">alias</span> <span style="color:#960050;background-color:#1e0010">&lt;</span><span style="color:#66d9ef">aliases</span><span style="color:#960050;background-color:#1e0010">&gt;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">action</span> <span style="color:#e6db74">&#34;outbound&#34;</span> <span style="color:#66d9ef">relay</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Actions for recieving mail +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">match</span> <span style="color:#66d9ef">from</span> <span style="color:#66d9ef">any</span> <span style="color:#66d9ef">for</span> <span style="color:#66d9ef">domain</span> <span style="color:#e6db74">&#34;example.com&#34;</span> <span style="color:#66d9ef">action</span> <span style="color:#e6db74">&#34;local&#34;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">match</span> <span style="color:#66d9ef">from</span> <span style="color:#66d9ef">local</span> <span style="color:#66d9ef">for</span> <span style="color:#66d9ef">local</span> <span style="color:#66d9ef">action</span> <span style="color:#e6db74">&#34;local&#34;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#75715e"># Actions for sending mail +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">match</span> <span style="color:#66d9ef">from</span> <span style="color:#66d9ef">any</span> <span style="color:#66d9ef">auth</span> <span style="color:#66d9ef">for</span> <span style="color:#66d9ef">any</span> <span style="color:#66d9ef">action</span> <span style="color:#e6db74">&#34;outbound&#34;</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">match</span> <span style="color:#66d9ef">from</span> <span style="color:#66d9ef">local</span> <span style="color:#66d9ef">for</span> <span style="color:#66d9ef">any</span> <span style="color:#66d9ef">action</span> <span style="color:#e6db74">&#34;outbound&#34;</span> +</span></span></code></pre></div><p>Now we need to place our mail delievery address in <code>/etc/mail/mailname</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">mail.example.com</span> +</span></span></code></pre></div><p>And now we can test our <code>smtpd</code> config by running:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas smtpd -n -f /etc/mail/smtpd.conf +</span></span></code></pre></div><p>If everything is all good then you can restart <code>smtpd</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas rcctl restart smtpd +</span></span></code></pre></div><h3 id="imappop3">IMAP/POP3</h3> +<p>The last part of our email server is <code>dovecot</code> which allows us to view our +emails using IMAP and or POP3. In order to use our certificates we will need to +configure <code>/etc/dovecot/conf.d/10-ssl.conf</code> like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">ssl_cert</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">&lt;</span>/etc/ssl/mail.example.com.fullchain.pem +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">ssl_key</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">&lt;</span>/etc/ssl/private/mail.example.com.key +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">ssl_dh</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">&lt;</span>/etc/dovecot/dh.pem +</span></span></code></pre></div><p>We need to generate a Diffie-Hellman key in order to make each TLS connection +unique (these commands might take a <em>while</em> to generate a key):</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas openssl dhparam -out /etc/dovecot/dh.pem <span style="color:#ae81ff">4096</span> +</span></span><span style="display:flex;"><span>doas chown _dovecot:_dovecot /etc/dovecot/dh.pem +</span></span><span style="display:flex;"><span>doas chmod <span style="color:#ae81ff">400</span> /etc/dovecot/dh.pem +</span></span></code></pre></div><p>Now we need to configure our mailbox and some basic settings for <code>dovecot</code>. Edit +<code>/etc/dovecot/conf.d/10-mail.conf</code> so it resembles the following:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#75715e"># Where to put user maildir +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span><span style="color:#66d9ef">mail_location</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">maildir</span><span style="color:#960050;background-color:#1e0010">:~</span>/Maildir +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">namespace</span> <span style="color:#66d9ef">inbox</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">inbox</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">yes</span> +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">mmap_disable</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">yes</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">first_valid_uid</span> <span style="color:#f92672">=</span> <span style="color:#ae81ff">1000</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">mail_plugin_dir</span> <span style="color:#f92672">=</span> /usr/local/lib/dovecot +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">protocol</span> <span style="color:#960050;background-color:#1e0010">!</span><span style="color:#66d9ef">indexer-worker</span> { +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">mbox_write_locks</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">fcntl</span> +</span></span></code></pre></div><p>Let&rsquo;s edit <code>/etc/dovecot/conf.d/20-lmtp.conf</code> in order to allow <code>sieve</code> and mail +plugins:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">protocol</span> <span style="color:#66d9ef">lmtp</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">mail_plugins</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">$</span><span style="color:#66d9ef">mail_plugins</span> <span style="color:#66d9ef">sieve</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>We now meed to modify <code>/etc/dovecot/conf.d/15-mailboxes.conf</code> and add this +inside the <code>namespace inbox {</code> section of the file in order to create a Spam +folder:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">mailbox</span> <span style="color:#66d9ef">Spam</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">auto</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">create</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">special_use</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">\</span><span style="color:#66d9ef">Junk</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>If you don&rsquo;t plan on using IMAP and only wish to use POP3 for viewing/sending +emails then this last step is entirely optional, however I encourage users to +use IMAP instead of POP3. To enable IMAP we just need to edit +<code>/etc/dovecot/conf.d/20-imap.conf</code> like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">protocol</span> <span style="color:#66d9ef">imap</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">mail_plugins</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">$</span><span style="color:#66d9ef">mail_plugins</span> <span style="color:#66d9ef">imap_sieve</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">mail_max_userip_connections</span> <span style="color:#f92672">=</span> <span style="color:#ae81ff">25</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><h4 id="sieve-and-filtering">Sieve and Filtering</h4> +<p>Finally, the last part of setting up <code>dovecot</code> is configuring <code>sieve</code> in order +to filter and group emails properly. Go ahead and edit +<code>/etc/dovecot/conf.d/90-plugin.conf</code> so it looks like the following:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">plugin</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">sieve_plugins</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">sieve_imapsieve</span> <span style="color:#66d9ef">sieve_extprograms</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#75715e"># Moving to Spam +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">1</span><span style="color:#66d9ef">_name</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">Spam</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">1</span><span style="color:#66d9ef">_causes</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">COPY</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">1</span><span style="color:#66d9ef">_before</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">file</span><span style="color:#960050;background-color:#1e0010">:</span>/usr/local/lib/dovecot/sieve/report-spam.sieve +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#75715e"># Moving from Spam +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">2</span><span style="color:#66d9ef">_name</span> <span style="color:#f92672">=</span> <span style="color:#960050;background-color:#1e0010">*</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">2</span><span style="color:#66d9ef">_from</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">Spam</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">2</span><span style="color:#66d9ef">_causes</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">COPY</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">imapsieve_mailbox</span><span style="color:#ae81ff">2</span><span style="color:#66d9ef">_before</span> <span style="color:#f92672">=</span> <span style="color:#66d9ef">file</span><span style="color:#960050;background-color:#1e0010">:</span>/usr/local/lib/dovecot/sieve/report-ham.sieve +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">sieve_pipe_bin_dir</span> <span style="color:#f92672">=</span> /usr/local/lib/dovecot/sieve +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">sieve_global_extensions</span> <span style="color:#f92672">=</span> <span style="color:#f92672">+</span><span style="color:#66d9ef">vnd.dovecot.pipe</span> <span style="color:#f92672">+</span><span style="color:#66d9ef">vnd.dovecot.environment</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>This configuration allows sieve to &ldquo;learn&rdquo; what types of emails are spam and +what types of messages are not spam (sometimes called &ldquo;ham&rdquo; ๐Ÿ–) depending on +where you move them. In order to make these filters functional we need to create +two sieve scripts. The first one is +<code>/usr/local/lib/dovecot/sieve/report-spam.sieve</code>. Let&rsquo;s create it and add the +following to it:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sieve" data-lang="sieve"><span style="display:flex;"><span><span style="color:#f92672">require</span> [<span style="color:#e6db74">&#34;vnd.dovecot.pipe&#34;</span>, <span style="color:#e6db74">&#34;copy&#34;</span>, <span style="color:#e6db74">&#34;imapsieve&#34;</span>, <span style="color:#e6db74">&#34;environment&#34;</span>, <span style="color:#e6db74">&#34;variables&#34;</span>]; +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span>if <span style="color:#960050;background-color:#1e0010">environment</span> <span style="color:#f92672">:matches</span> <span style="color:#e6db74">&#34;imap.user&#34;</span> <span style="color:#e6db74">&#34;*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">set</span> <span style="color:#e6db74">&#34;username&#34;</span> <span style="color:#e6db74">&#34;${1}&#34;</span>; +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#960050;background-color:#1e0010">pipe</span> <span style="color:#f92672">:copy</span> <span style="color:#e6db74">&#34;sa-learn-spam.sh&#34;</span> [ <span style="color:#e6db74">&#34;${username}&#34;</span> ]; +</span></span></code></pre></div><p>The second filter is <code>/usr/local/lib/dovecot/sieve/report-ham.sieve</code> and should +look like this:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sieve" data-lang="sieve"><span style="display:flex;"><span><span style="color:#f92672">require</span> [<span style="color:#e6db74">&#34;vnd.dovecot.pipe&#34;</span>, <span style="color:#e6db74">&#34;copy&#34;</span>, <span style="color:#e6db74">&#34;imapsieve&#34;</span>, <span style="color:#e6db74">&#34;environment&#34;</span>, <span style="color:#e6db74">&#34;variables&#34;</span>]; +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span>if <span style="color:#960050;background-color:#1e0010">environment</span> <span style="color:#f92672">:matches</span> <span style="color:#e6db74">&#34;imap.mailbox&#34;</span> <span style="color:#e6db74">&#34;*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">set</span> <span style="color:#e6db74">&#34;mailbox&#34;</span> <span style="color:#e6db74">&#34;${1}&#34;</span>; +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span>if string <span style="color:#e6db74">&#34;${mailbox}&#34;</span> <span style="color:#e6db74">&#34;Trash&#34;</span> { +</span></span><span style="display:flex;"><span> stop; +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span>if <span style="color:#960050;background-color:#1e0010">environment</span> <span style="color:#f92672">:matches</span> <span style="color:#e6db74">&#34;imap.user&#34;</span> <span style="color:#e6db74">&#34;*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">set</span> <span style="color:#e6db74">&#34;username&#34;</span> <span style="color:#e6db74">&#34;${1}&#34;</span>; +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#960050;background-color:#1e0010">pipe</span> <span style="color:#f92672">:copy</span> <span style="color:#e6db74">&#34;sa-learn-ham.sh&#34;</span> [ <span style="color:#e6db74">&#34;${username}&#34;</span> ]; +</span></span></code></pre></div><p>Now we will create two shell scripts for these filters. The first one is +<code>/usr/local/lib/dovecot/sieve/sa-learn-ham.sh</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span><span style="color:#75715e">#!/bin/sh +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span>exec /usr/local/bin/rspamc -d <span style="color:#e6db74">&#34;</span><span style="color:#e6db74">${</span>1<span style="color:#e6db74">}</span><span style="color:#e6db74">&#34;</span> learn_ham +</span></span></code></pre></div><p>The second one is <code>/usr/local/lib/dovecot/sieve/sa-learn-spam.sh</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span><span style="color:#75715e">#!/bin/sh +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span>exec /usr/local/bin/rspamc -d <span style="color:#e6db74">&#34;</span><span style="color:#e6db74">${</span>1<span style="color:#e6db74">}</span><span style="color:#e6db74">&#34;</span> learn_spam +</span></span></code></pre></div><p>And we need to make them executable:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas chmod +x /usr/local/lib/dovecot/sieve/sa-learn-spam.sh /usr/local/lib/dovecot/sieve/sa-learn-ham.sh +</span></span></code></pre></div><p>Now we need to compile the sieve filters to actually use them:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas sievec /usr/local/lib/dovecot/sieve/report-spam.sieve +</span></span><span style="display:flex;"><span>doas sievec /usr/local/lib/dovecot/sieve/report-ham.sieve +</span></span></code></pre></div><p>Lastly, let&rsquo;s enable and start <code>dovecot</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas rcctl enable dovecot +</span></span><span style="display:flex;"><span>doas rcctl start dovecot +</span></span></code></pre></div><h2 id="done">Done!</h2> +<p><strong>Congrats!</strong> You now have your own email server running on OpenBSD! You can now +login to your email server using your preferred email client. If you don&rsquo;t have +a preferred email client then you can use <a href="https://thunderbird.net">Thunderbird</a> +if you want a GUI or <a href="https://aerc-mail.org/">aerc</a> if you want to view emails +from the terminal.</p> +<p>You can now enjoy using email knowing that your inbox won&rsquo;t be monitored. ๐Ÿ˜Ž</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>Just a friendly reminder to change &ldquo;example.com&rdquo; to your specific domain +name.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + OpenBSD Server: httpd + https://brycev.com/blog/openbsd-httpd/ + Thu, 27 Feb 2025 00:00:00 +0000 + https://brycev.com/blog/openbsd-httpd/ + <p>I finally got around to writing part 2 of this series. If you have not read part +1 yet, then you can find it <a href="https://brycev.com/blog/openbsd-server-install">here</a>. So now that we +have our base server installed, we can go ahead and setup <code>httpd</code> to serve our +website.</p> +<h2 id="setup-httpd">Setup httpd</h2> +<p>Let&rsquo;s go ahead and create a very simple http server using <code>httpd</code> just to make +sure that everything is working properly. We can make a config file for <code>httpd</code> +at <code>/etc/httpd.conf</code> like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">server</span> <span style="color:#e6db74">&#34;example.com&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#960050;background-color:#1e0010">*</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">80</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">alias</span> <span style="color:#e6db74">&#34;www.example.com&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">root</span> <span style="color:#e6db74">&#34;/htdocs/example.com&#34;</span> +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">types</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">include</span> <span style="color:#e6db74">&#34;/usr/share/misc/mime.types&#34;</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>This will create a web server for &ldquo;example.com&rdquo; (make sure to replace +&ldquo;example.com&rdquo; with your domain name) on port 80 and will redirect traffic from +&ldquo;<a href="https://www.example.com">www.example.com</a>&rdquo; to &ldquo;example.com&rdquo;. Next we need to create +<code>/var/www/htdocs/example.com</code> to host our files:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>mkdir -p /var/www/htdocs/example.com +</span></span></code></pre></div><p>Let&rsquo;s just put a very basic webpage at <code>/var/www/htdocs/example.com/index.html</code> +to serve as our homepage.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-html" data-lang="html"><span style="display:flex;"><span><span style="color:#75715e">&lt;!DOCTYPE html&gt;</span> +</span></span><span style="display:flex;"><span>&lt;<span style="color:#f92672">html</span> <span style="color:#a6e22e">lang</span><span style="color:#f92672">=</span><span style="color:#e6db74">&#34;en&#34;</span>&gt; +</span></span><span style="display:flex;"><span> &lt;<span style="color:#f92672">head</span>&gt; +</span></span><span style="display:flex;"><span> &lt;<span style="color:#f92672">title</span>&gt;Example webpage&lt;/<span style="color:#f92672">title</span>&gt; +</span></span><span style="display:flex;"><span> &lt;<span style="color:#f92672">meta</span> <span style="color:#a6e22e">charset</span><span style="color:#f92672">=</span><span style="color:#e6db74">&#34;utf-8&#34;</span>&gt; +</span></span><span style="display:flex;"><span> &lt;/<span style="color:#f92672">head</span>&gt; +</span></span><span style="display:flex;"><span> &lt;<span style="color:#f92672">body</span>&gt; +</span></span><span style="display:flex;"><span> &lt;<span style="color:#f92672">h1</span>&gt;This is an example webpage&lt;/<span style="color:#f92672">h1</span>&gt; +</span></span><span style="display:flex;"><span> &lt;<span style="color:#f92672">p</span>&gt;Looks like it works!&lt;/<span style="color:#f92672">p</span>&gt; +</span></span><span style="display:flex;"><span> &lt;/<span style="color:#f92672">body</span>&gt; +</span></span><span style="display:flex;"><span>&lt;/<span style="color:#f92672">html</span>&gt; +</span></span></code></pre></div><p>Now, in order to make sure that our httpd config file is correct and does not +have any errors, we can run this command to check it:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas httpd -n +</span></span></code></pre></div><p>If your configuration is correct, then it will spit out <code>configuration ok</code> and +you can move on. If not, then it will spit out what error you have to fix in the +<code>/etc/httpd.conf</code> file. Finally we can enable httpd on boot and start it using +the <code>rcctl</code> command:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas rcctl enable httpd +</span></span><span style="display:flex;"><span>doas rcctl start httpd +</span></span></code></pre></div><p><strong>Congrats!</strong> You now have a web page on the internet at &ldquo;example.com&rdquo;!</p> +<h2 id="getting-a-certificate">Getting a certificate</h2> +<p>Now that we verified that a normal web server works, we can go ahead and setup +ACME in order to obtain a certificate in order to allow HTTPS connects to our +website. First we need to create a file named <code>/etc/acme-client.conf</code> and add +this to our file:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">authority</span> <span style="color:#66d9ef">letsencrypt</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">api</span> <span style="color:#66d9ef">url</span> <span style="color:#e6db74">&#34;https://acme-v02.api.letsencrypt.org/directory&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">account</span> <span style="color:#66d9ef">key</span> <span style="color:#e6db74">&#34;/etc/acme/letsencrypt-privkey.pem&#34;</span> +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">domain</span> <span style="color:#66d9ef">example.com</span> { +</span></span><span style="display:flex;"><span> <span style="color:#75715e"># Allows an alias +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span> <span style="color:#66d9ef">alternative</span> <span style="color:#66d9ef">names</span> { <span style="color:#66d9ef">www.example.com</span> } +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">domain</span> <span style="color:#66d9ef">key</span> <span style="color:#e6db74">&#34;/etc/ssl/private/example.com.key&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">domain</span> <span style="color:#66d9ef">full</span> <span style="color:#66d9ef">chain</span> <span style="color:#66d9ef">certificate</span> <span style="color:#e6db74">&#34;/etc/ssl/example.com.fullchain.pem&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">sign</span> <span style="color:#66d9ef">with</span> <span style="color:#66d9ef">letsencrypt</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>The first section defines Let&rsquo;s Encrypt as our certificate authority. The +second section defines a domain to fetch a certificate for. It will place our +domain key at <code>/etc/ssl/private/example.com.key</code> and our certificate at +<code>/etc/ssl/example.com.fullchain.pem</code> (remember to replace &ldquo;example.com&rdquo; with +your domain).</p> +<p>Now we have to edit <code>/etc/httpd.conf</code> in order to ACME to generate a certificate +for our domain:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">server</span> <span style="color:#e6db74">&#34;example.com&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#960050;background-color:#1e0010">*</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">80</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">alias</span> <span style="color:#e6db74">&#34;www.example.com&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">root</span> <span style="color:#e6db74">&#34;/htdocs/example.com&#34;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">location</span> <span style="color:#e6db74">&#34;/.well-known/acme-challenge/*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">request</span> <span style="color:#66d9ef">strip</span> <span style="color:#ae81ff">2</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">root</span> <span style="color:#e6db74">&#34;/acme&#34;</span> +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">types</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">include</span> <span style="color:#e6db74">&#34;/usr/share/misc/mime.types&#34;</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>Now we can run this command in order to generate a certificate for our domain:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>acme-client -v example.com +</span></span></code></pre></div><h2 id="setup-https">Setup HTTPS</h2> +<p>Now that we have our certificate, let&rsquo;s go ahead and edit our <code>/etc/httpd.conf</code> +file in order to use HTTPS for our web server:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-lighty" data-lang="lighty"><span style="display:flex;"><span><span style="color:#66d9ef">server</span> <span style="color:#e6db74">&#34;example.com&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#960050;background-color:#1e0010">*</span> <span style="color:#66d9ef">tls</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">443</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">alias</span> <span style="color:#e6db74">&#34;www.example.com&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">root</span> <span style="color:#e6db74">&#34;/htdocs/example.com&#34;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">tls</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">certificate</span> <span style="color:#e6db74">&#34;/etc/ssl/example.com.fullchain.pem&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">key</span> <span style="color:#e6db74">&#34;/etc/ssl/private/example.com.key&#34;</span> +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">location</span> <span style="color:#e6db74">&#34;/.well-known/acme-challenge/*&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">request</span> <span style="color:#66d9ef">strip</span> <span style="color:#ae81ff">2</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">root</span> <span style="color:#e6db74">&#34;/acme&#34;</span> +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">server</span> <span style="color:#e6db74">&#34;example.com&#34;</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">listen</span> <span style="color:#66d9ef">on</span> <span style="color:#960050;background-color:#1e0010">*</span> <span style="color:#66d9ef">port</span> <span style="color:#ae81ff">80</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">alias</span> <span style="color:#e6db74">&#34;www.example.com&#34;</span> +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">block</span> <span style="color:#66d9ef">return</span> <span style="color:#ae81ff">301</span> <span style="color:#e6db74">&#34;https://example.com$REQUEST_URI&#34;</span> +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">types</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">include</span> <span style="color:#e6db74">&#34;/usr/share/misc/mime.types&#34;</span> +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>Finally we can restart <code>httpd</code> and go to &ldquo;example.com&rdquo; and we will have a secure +connection to the website.</p> +<h2 id="automatically-renew-certificates">Automatically renew certificates</h2> +<p>The final part of this post is going to take care of lose ends by automatically +renewing our certificate before it expires. Most certificates expire after only +a few months, so we want to automatically renew them. In order to do so we can +make a new file at <code>/etc/weekly.local</code>. Any commands that we put in this file +will run once a week, so lets put this in <code>/etc/weekly.local</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span><span style="color:#75715e"># Check for new certificates every week</span> +</span></span><span style="display:flex;"><span>acme-client -v example.com +</span></span><span style="display:flex;"><span>rcctl restart httpd +</span></span></code></pre></div><h2 id="almost-done">Almost Done</h2> +<p>That&rsquo;s the end of part 2. +The final part of this 3 part series will be about setting up an email server +using <code>smtpd</code> and setting up spam filtering (as well as other things). So stay +tuned for part 3.</p> +<hr> +<p>You can now read part 3 of this series <a href="https://brycev.com/blog/openbsd-smtpd">here</a>.</p> +<hr> + + + + + OpenBSD Server: Initial Setup + https://brycev.com/blog/openbsd-server-install/ + Thu, 13 Feb 2025 00:00:00 +0000 + https://brycev.com/blog/openbsd-server-install/ + <p><em>This is the first part in a series of posts. Find <a href="https://brycev.com/blog/openbsd-httpd">part 2</a> +and <a href="https://brycev.com/blog/openbsd-smtpd">part 3</a> when they are ready</em></p> +<hr> +<p>For those of you who don&rsquo;t follow my blog, in my +<a href="https://brycev.com/blog/migrating-my-vps">previous blog post</a> I talked about migrating my VPS to +use OpenBSD instead of Debian stable. I recently finished migrating my VPS and +would like to document the process in order to help anyone setting up their own +OpenBSD server or VPS.</p> +<p>Before we get started, make sure that you have your DNS records for your domain +name pointing to your server IP address.</p> + + +<figure ><img src="https://brycev.com/p/openbsd.webp" title="OpenBASED" alt="OpenBSD logo"></figure> + +<h2 id="installing-openbsd">Installing OpenBSD</h2> +<p>One thing that might stop most people from using OpenBSD on a VPS specifically +is the fact that most hosting providers <strong>don&rsquo;t</strong> allow you to custom +operating systems besides the ones that they offer. Luckily, my VPS provider +<a href="https://my.frantech.ca/aff.php?aff=6418">Frantech</a><sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>, allows me to upload my +own ISO images in order for you to install <strong>any</strong> OS that I like. So make sure +that your VPS provider actually allows you to run OpenBSD.</p> +<p>Go ahead and install OpenBSD normally. Here are a few tips when installing +OpenBSD for a server:</p> +<ul> +<li>Make sure to start <code>sshd</code> by default (otherwise you can&rsquo;t login)</li> +<li>Create a user for your system</li> +<li>Make sure you allow root ssh login (we&rsquo;ll need it temporarily)</li> +<li>You can delete the <code>swap</code>, <code>/usr/X11R6</code>, <code>/usr/src/</code>, and <code>/usr/obj</code> +partitions when partitioning your disk as we don&rsquo;t need them (make sure to use +that free space though)</li> +<li>Make sure you select to install <strong>all</strong> sets unless you know what you&rsquo;re +doing</li> +</ul> +<p><strong>Congrats!</strong> You now have a basic OpenBSD installation.</p> +<h2 id="configuring">Configuring</h2> +<p>First thing is first, we need root privileges. So run this in order to connect +as the root user:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>ssh root@example.com +</span></span></code></pre></div><p>where <code>example.com</code> is your domain name. As the root user we need to create the +<code>/etc/doas.conf</code> file and add the wheel group to it. We can do so by running:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>echo <span style="color:#e6db74">&#34;permit persist :wheel&#34;</span> &gt; /etc/doas.conf +</span></span></code></pre></div><p>We can now exit our current ssh session in order to login as the user we created +when we installed OpenBSD.</p> +<p>Let&rsquo;s first update our system by running these commands using <code>doas</code>:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>doas syspatch +</span></span><span style="display:flex;"><span>doas pkg_add -Uu +</span></span></code></pre></div><p>The <code>syspatch</code><sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup> command applies any necessary patches to your system and <code>pkg_add -Uu</code> just updates packages. Speaking of adding packages, you can use <code>pkg_add</code> +in order to install your favorite text editor (you&rsquo;ll need it). +While we are at it, let&rsquo;s go ahead and disable ssh +logins for root since we don&rsquo;t need it anymore. Edit <code>/etc/ssh/sshd_config</code> and +add this line to it:</p> +<pre tabindex="0"><code>PermitRootLogin no +</code></pre><p>Finally, let&rsquo;s create the file <code>/etc/sysctl.conf</code> that will contain system-wide +settings. We will put this in <code>/etc/sysctl.conf</code>:</p> +<pre tabindex="0"><code>kern.maxproc=8192 +kern.maxfiles=32768 +kern.maxthread=16384 +kern.shminfo.shmall=536870912 +kern.shminfo.shmmax=2147483647 +kern.shminfo.shmmni=4096 +</code></pre><p>Please note that these settings are for <strong>my</strong> system and will chage from system +to system. As a rule of thumb:</p> +<ul> +<li><code>kern.shminfo.shmmax</code> should be set to <strong>half</strong> of your maximum RAM (in bytes)</li> +<li><code>kern.shminfo.shmall</code> should be set to <code>kern.shminfo.shmmax</code> divided by 4096 +(4096 is the page size in OpenBSD).</li> +</ul> +<p>These are just a few settings that will get a bit more performance out of our +OpenBSD system. It just increases the maximum number of processes able to run at +once as well as the maximum amount of shared memory. +You can find a detailed explanation of these settings by looking +at the man page for <code>sysctl</code> (hint: run <code>man 2 sysctl</code>).</p> +<h2 id="stay-tuned">Stay tuned</h2> +<p>That&rsquo;s just the beginning of setting up an OpenBSD server, but keep a close eye +on my blog as I will upload the next part of this guide somewhat soon which will +be about setting up a static web server using <code>httpd</code>&hellip;assuming that I get +around to writing it eventually.</p> +<hr> +<p>You can now read part 2 of this series <a href="https://brycev.com/blog/openbsd-httpd">here</a>.</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>This is an affiliate link&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>Make sure to reboot whenever <code>syspatch</code> applies patches in order to reload +the OpenBSD kernel.&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + Migrating My VPS + https://brycev.com/blog/migrating-my-vps/ + Fri, 07 Feb 2025 00:00:00 +0000 + https://brycev.com/blog/migrating-my-vps/ + <p>Please note that on February 11, 2025 I will be migrating my VPS to a +different operating system. This means that my website, email, and XMPP server +will be down for most of the day. If you need to contact me please wait until +Feburary 12, 2025.</p> +<p>I&rsquo;ve been using Debian stable on my VPS ever since I +wrote my <a href="https://brycev.com/blog/i-finally-have-a-vps">original post</a> about getting a VPS, +however I have decided that I would like to use +<a href="https://www.openbsd.org/">OpenBSD</a> on my VPS instead. OpenBSD offers more +security while still being very simple which fits my needs a lot more for a +server operating system when compared to Debian stable.</p> +<p>After migrating my system, I&rsquo;ll most likely create a guide on how to setup an +OpenBSD system as it&rsquo;s pretty easy, but not a well documented process.</p> + + + + + First Livestream Soon + https://brycev.com/blog/first-livestream-soon/ + Sat, 03 Aug 2024 00:00:00 +0000 + https://brycev.com/blog/first-livestream-soon/ + <p>I&rsquo;ve recently decided that I&rsquo;m going to do a livestream. This livestream is +mostly just a test, but feel free to stop on by if you want to. +I have no idea what to expect&hellip;so I&rsquo;ll see how this goes. +The stream will start on August 7 at 5:30 PM (EST).</p> +<p>The link is here: +<a href="https://www.youtube.com/watch?v=y5eZXwXIrBI">https://www.youtube.com/watch?v=y5eZXwXIrBI</a></p> + + + + + Microwave Transformers + https://brycev.com/blog/microwave-transformers/ + Sat, 20 Apr 2024 00:00:00 +0000 + https://brycev.com/blog/microwave-transformers/ + <p>A few years ago I was (and I still am) interested in electrical engineering +and learning about how electricity works. I built circuits and repaired +various different electronic devices around my house that were broken. +My favorite part however, was working with high voltage. The cheapest and +most effective gateway drug into high voltage that I encountered was actually +the humble microwave oven.</p> +<h2 id="the-guts-of-a-microwave-oven">The Guts of a Microwave Oven</h2> +<p>Believe it or not, microwave ovens are actually deceptively simple. It really +consists of 3 main parts:</p> +<ul> +<li>The control panel</li> +<li>The magnetron</li> +<li>The high voltage transformer</li> +</ul> +<p>The control panel is pretty self explanatory, it lets the +user input the time into the microwave oven, change settings, and etc. +In fact, a control panel isn&rsquo;t <em>really</em> necessary, all that the control +panel does is turn the transformer on and off, which could potentially be +done manually. +The magnetron is the part that actually generates the microwave radiation +needed to cook your food. The high voltage transformer is what provides the +necessary power for the magnetron to actually work. The transformer is what +we will be focusing on.</p> +<h2 id="the-transformer">The Transformer</h2> +<p>If you&rsquo;re unfamiliar with transformers and how they work, I would recommend +learning a bit about them <a href="http://hyperphysics.phy-astr.gsu.edu/hbase/magnetic/transf.html">here</a>. +Otherwise, let&rsquo;s continue.</p> +<p>The transformer within a microwave oven is very special. Most high voltage +transformers that you can easily get a hold of are very&hellip;limited. Usually +these transformers output a high voltage (around 5,000 volts max), but have at +a pretty low current (less than 5 milliamps). While this can potentially hurt +you and leave burns, it is usually not enough to kill you. Microwave oven +transformers are <strong>very</strong> different.</p> +<p>While microwave oven transformers are pretty easy to get a hold of, they can +output a <strong>scary</strong> amount of power. The average microwave oven transformer can +output anywhere from 2,000 to 4,000 volts at around <strong>500 milliamps</strong> (or 0.5 +amps). That&rsquo;s around <strong>1-2 thousand watts</strong> of power!</p> +<h2 id="some-fun">Some Fun</h2> +<p>With high voltage like this, you can do some really cool stuff. +But first, a disclaimer:</p> +<blockquote> +<h3 id="disclaimer">Disclaimer</h3> +<p>This should go without saying, but do <strong>NOT</strong> work with microwave ovens and +microwave oven transformers unless you know 100% what you are doing! +These things can <em>easily</em> kill you.</p> +<p>Human skin is usually 100 killoohms, this means that if you touch even a +2,000 volt microwave transformer, then you could have at <em>least</em> 20 +milliamps flow through your body<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>. This is enough current to seize your +muscles so you can&rsquo;t let go. ๐Ÿ’€</p> +<p>I am <strong>NOT</strong> responsible for anything stupid that you do.</p></blockquote> +<p>This is the transformer that I pulled out of a microwave oven. Like most +transformers there&rsquo;s two coils, the primary and the secondary<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup>. +These are illustrated below:</p> + + +<figure ><img src="https://brycev.com/p/transformer.webp" alt="Mircowave Transformer"></figure> + +<p>By hooking up the primary side of the transformer to 120 volts AC (a.k.a. +Mains electricity) and attaching wires to the secondary side, I was able to get +some pretty cool arcs. I achieved this by bridging the gap between the +secondary side with a nail. This setup is called a +<a href="https://teslauniverse.com/build/plans/jacobs-ladder">Jacob&rsquo;s Ladder</a>.</p> +<video width="640" height="480" preload="none" controls> + <source src="https://brycev.com/ac-arcs.webm" type="video/webm"> +</video> + +<p>We can do better. We can build a self-sustaining (non-manual) Jacob&rsquo;s +Ladder&hellip;but how?</p> +<h2 id="more-power">More Power</h2> +<p>In order to create a self-sustaining and self-starting Jacob&rsquo;s Ladder I needed +more power. But how exactly can we add more power to this circuit? One +microwave oven transformer already draws a lot of power, but how about <strong>two</strong> +microwave oven transformers? That&rsquo;s right, I got a hold of <em>another</em> microwave +oven transformer.</p> +<p>Careful consideration is needed in order to hook up two of these microwave oven +transformers. I wired the two transformers in series with a &ldquo;center tap&rdquo; going +straight to ground (otherwise my breaker would trip). This gives me anywhere +from 4,000 volts to 8,000 volts AC. Next I need to turn that AC voltage into +DC voltage.</p> +<p>Sadly, I did not have enough high voltage components to make a <em>true</em> AC to +DC conversion system&hellip;but I hobbled one together anyways. I was able to +use two microwave oven capacitors as resistors<sup id="fnref:3"><a href="#fn:3" class="footnote-ref" role="doc-noteref">3</a></sup> since I didn&rsquo;t have any +high power resistors on hand. In order to turn AC into &ldquo;DC&rdquo; I took and very +simple [but stupid] approach. During the negative AC cycle, all the current +is dumped into a pair of diodes&hellip;that&rsquo;s it. Does it waste a <strong>lot</strong> of power? +Yes. But does it work? <strong>Yes</strong>. I made a schematic below in order to better +illustrate the circuit.</p> + + +<figure ><a href="https://brycev.com/p/circuit.webp"><img src="https://brycev.com/p/circuit.webp" alt="Jacob&#39;s Ladder circuit"></a></figure> + +<p>And here is the final product. A <strong>working</strong> Jacob&rsquo;s Ladder!</p> +<video width="640" height="480" preload="none" controls> + <source src="https://brycev.com/dc-arcs.webm" type="video/webm"> +</video> + +<p>I&rsquo;m not sure if you noticed in that video, but my breaker popped. This circuit +just draws too much power. I could improve this circuit, but I think I&rsquo;ve +decided to end my project here. +That being said, I think this was enough high +voltage fun for one day. :)</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>These are actually very conservative estimates. In reality, much more +current is likely to flow through your body&hellip;which would kill you even faster.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>Most microwave oven transformers actually have a third coil (see the +image). +This coil usually generates a very low voltage with a high current capability. +This is used to heat the magnetron filament, however we are not concerned with +it.&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:3"> +<p>For those who don&rsquo;t know, capacitors behave similar to resistors when +passing AC current through them due to +<a href="http://hyperphysics.phy-astr.gsu.edu/hbase/electric/accap.html#c2">capacitive reactance</a>.&#160;<a href="#fnref:3" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + My Conversion + https://brycev.com/blog/my-conversion/ + Mon, 11 Mar 2024 00:00:00 +0000 + https://brycev.com/blog/my-conversion/ + <p>I&rsquo;ve been putting off writing this blog post for 2 (almost 3) years now. +There <strong>is</strong> a good reason for why I did that. Part of it was that I didn&rsquo;t +know how to put my experience into words until now, and the other part +was just procrastination. So now I feel like I finally have the experience +and knowledge to write about something that is very important to me.</p> +<h2 id="a-bit-of-background">A Bit of Background</h2> +<p>In the past I have never really been a religious person, nor have I really +ever liked religion. I always either found it silly, meaningless, or something +between those two. In high school I was something akin to Agnostic, I didn&rsquo;t +fully believe in God, but I wasn&rsquo;t really sure. +In middle school I was a proud Atheist who had a smug +satisfaction &ldquo;knowing&rdquo; that God did not exist. So take my word when I say the +events within the past few years of my life completely took me by surprise.</p> +<h2 id="i-am-now-a-roman-catholic">I Am Now A Roman Catholic</h2> + + +<figure ><img src="https://brycev.com/p/augustine.webp" title="Based Saint Augustine of Hippo" alt="Saint Augustine of Hippo"></figure> + +<p>Ok, <em>technically</em> I&rsquo;ve been a Catholic for the last 2-3 years and I got +confirmed last year, but I&rsquo;ve been +putting off &ldquo;officially&rdquo; saying anything until now. I was actually considering +Catholicism when I originally started this blog all the way back in November of +2021 (my <a href="https://brycevandegrift.xyz/blog/new-blog/">first blog post</a>), but I +didn&rsquo;t start taking it seriously until sometime in 2022. +For me, it seemed like it all happened very quickly.</p> +<blockquote> +<p>&ldquo;But did you consider religion <em>xyz</em>?&rdquo; +&ndash;Probably Someone</p></blockquote> +<p>I actually did consider quite a few. I considered different forms of +Christianity like Anglicanism, Orthodoxy, and even Baptist<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>. I even +considered things like Islam, Buddhism, and even Hinduism briefly. However, I +never found a intellectual tradition that truly rivaled Catholic tradition<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup>. +I also personally looked for the religion that I thought was 100% <strong>true</strong> and +Catholicism seemed to fit that bill. +It should go without saying, but +basing your religious decisions on anything else besides truth (like personal +preference) is a very small brained move.</p> +<p>I didn&rsquo;t really have a bias either. I used to <strong>hate</strong> the Catholic Church, but +I soon grew to find out that my hatred was <em>very</em> much misplaced. As Venerable +Archbishop Fulton Sheen put it:</p> +<blockquote> +<p>&ldquo;Not 100 people in the United States hate the Roman Catholic Church, but +millions hate what they think the Roman Catholic Church is.&rdquo; &ndash;Archbishop +Fulton J. Sheen</p></blockquote> +<p>This very much encapsulates the essence of what stopped me from converting to +Catholicism in the past. I hated what I <em>thought</em> Catholicism was, but I soon +came to love what it <em>actually</em> is.</p> +<h2 id="joining-a-webring">Joining a Webring</h2> +<p>After being invited to the <a href="https://heaventree.xyz">Heaven Tree</a> webring almost +a year ago, I decided to accept the invitation last month. Now I didn&rsquo;t wait a +year to accept the invitation just to be rude. I waited quite a while because I +was not sure if I really wanted to identify as a Catholic or even as a Christian +for that matter. It really took me the last year to solidify my identity as a +Catholic and to accept it. So I sort of think that joining this webring is +me cementing myself as a Catholic.</p> +<h2 id="what-now">What Now?</h2> +<p>Not much about my blog or YouTube channel will change&hellip;at least in the short +term. I might post more stuff about theology or other topics related to it, but +as always I don&rsquo;t really have a specific direction for my blog or channel. I +just post about whatever I feel like saying and that&rsquo;s the way it will probably +always be. So don&rsquo;t be surprised if things change over time.</p> +<p>I am very grateful for my conversion and all that it has brought me. DEO GRATIAS.</p> +<blockquote> +<p>&ldquo;Late have I loved you, Beauty so ancient and so new, late have I loved you!</p> +<p>Lo, you were within, +but I outside, seeking there for you, +and upon the shapely things you have made +I rushed headlong โ€“ I, misshapen. +You were with me, but I was not with you. +They held me back far from you, +those things which would have no being, +were they not in you.</p> +<p>You called, shouted, broke through my deafness; +you flared, blazed, banished my blindness; +you lavished your fragrance, I gasped; and now I pant for you; +I tasted you, and now I hunger and thirst; +you touched me, and I burned for your peace.&rdquo;</p> +<p>&ndash;Saint Augustine of Hippo</p></blockquote> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>The only reason I even <strong>remotely</strong> considered Baptist was due to their +incredible zeal. Baptist theology is extremely deficient and almost +non-sensical.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>I mean, have you even looked at the Summa Theologica?&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + Using a Dot Matrix Printer in 2024 + https://brycev.com/blog/using-a-dot-matrix-printer-in-2024/ + Wed, 14 Feb 2024 00:00:00 +0000 + https://brycev.com/blog/using-a-dot-matrix-printer-in-2024/ + <p>Modern printers suck. They&rsquo;re either overpriced, unreliable, ink/toner hogs, +or a combination of all 3.</p> +<p>Inkjet printers are cheap, have a good picture quality, and can print on +many surfaces. However, they are also <strong>very</strong> unreliable and usually don&rsquo;t +last over 3 years. Not only that but inkjet printer cartridges can cost +and arm and a leg, which is dumb considering ink (outside the cartridge) +is dirt cheap. Inkwell printers do solve the ink issue by having an ink +reservoir which means that you are not trapped into buying DRM infested +ink cartridges.</p> +<p>Laser printers are reliable, have a <strong>great</strong> picture quality, and are +usually more serviceable. However, they are often <strong>very</strong> large and +sometimes overly complicated. Just like with inkjet printers, laser printer +toner is insanely expensive and can cost anywhere from $60-$140 per cartridge. +It doesn&rsquo;t help that +[good] laser printers themselves are also very expensive, costing anywhere +from $500-$2,000.</p> +<p>Both laser and inkjet printers also suffer from having &ldquo;internet&rdquo; connectivity +put on them. I&rsquo;m not talking about a simple LAN connection, I&rsquo;m talking +about having your printer connected to Hewlett Packard&rsquo;s, Canon&rsquo;s, or +Epson&rsquo;s servers over the internet. Obviously, I&rsquo;m not a big fan of having +my printer being able to relay information to anywhere outside my house +without my permission.</p> +<p>So, what are we left with after laser and inkjet printers. Well, there are +thermal printers which are really cheap, require no ink/toner, and can print +quickly. However, they require special paper, the prints are usually not high +quality, and since a thermal printer <em>burns</em> the image onto paper, the image +wears off the paper after a year or so. You also can&rsquo;t print in color.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup></p> +<p>I guess that means we are left with&hellip;</p> +<h2 id="dot-matrix-printers">Dot Matrix Printers</h2> +<p>Dot matrix printers are relatively cheap, usually costing no more than +$500. They&rsquo;re also simple since a dot matrix printer&rsquo;s method of printing is +relatively straight forward (I&rsquo;ll explain how they work in a moment). Since +they&rsquo;re simple, that also means that they&rsquo;re also <strong>very</strong> reliable and easy +to service. Dot matrix ink ribbons are also cheap compared to inkjet ink +and laser printer toner. There are also color options for dot matrix printers. +The only real apparent downside of dot matrix printers is speed as it usually +takes a while to print compared to other printers.<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup></p> +<h2 id="how-they-work">How They Work</h2> +<p>If you&rsquo;re not familiar with how dot matrix printers work, you would be +surprised by how simple they are. A dot matrix printer, as the name implies, +prints in a matrix of dots. As the print head moves from left to right, it +presses the ink onto the paper via a series of tiny pins that create the +final image. It does this line by line until the entire picture is printed. +If you want to see one in action look <a href="https://youtu.be/A_vXA058EDY">here</a>.</p> + + +<figure ><img src="https://brycev.com/p/dotmatrix.webp" alt="How a dot matrix printer works"></figure> + +<p>Sadly, as you can tell from the video, this means that dot matrix printers +are somewhat slow and sometimes loud compared to their contenders.</p> +<h2 id="my-not-really-new-printer">My (Not Really) New Printer</h2> +<p>About a year or two ago I acquired an Apple ImageWriter II off of eBay for +about $50. It had some wear and tear on it and was <em>severely</em> yellowed (which +I don&rsquo;t mind) and it was also previously used in an industrial setting. But +despite all of that, it worked great! The only problem it had was that one +of the pins on the print head did not work, which you will see in a moment. +Considering there is only <strong>one</strong> problem with it after 30 years of industrial +use is amazing and really shows the reliability of these things.</p> +<p>I planned to use this thing for a while, but I kept on putting it off +for about 1-2 years. It wasn&rsquo;t until the last 2 weeks that I decided to +get this thing working.</p> + + +<figure ><img src="https://brycev.com/p/imagewriter.webp" title="The last good Apple product" alt="Apple Imagewriter II"></figure> + +<h2 id="getting-it-to-work">Getting It To Work</h2> +<h3 id="printing-text">Printing Text</h3> +<p>The first problem I encountered was how to even hook this thing up to my +modern computer. My desktop has a 9 pin serial card and the ImageWriter is a +serial printer, so I figured that I should start there. The printer came with a +cable to covert the 8 pin DIN socket on the printer to a 25 pin serial +connector, so I decided to buy a 25 pin to 9 pin null modem serial cable and a +25 pin serial coupler. I hooked it up as followed:</p> +<pre tabindex="0"><code> DB-25 Serial Port + | | +DB-9 Serial Port-------+ | | +-------DIN 8 Serial Port + | | | | + | | | | + +-----------+ | | +-------+ | | +---------+ + | | v v | | v v | | + | Computer +-+--+-+Coupler+-+-+-+ Printer | + | | | | | | + +-----------+ +-------+ +---------+ +</code></pre><p>Next, I had to get access to <code>/dev/ttyS1</code> which is where my serial port +connected to the printer is located. To do that I just created a simple udev +rule to change the permissions of <code>/dev/ttyS1</code> on boot.</p> +<pre tabindex="0"><code>KERNEL==&#34;ttyS1&#34;, GROUP=&#34;plugdev&#34;, MODE=&#34;0660&#34; +</code></pre><p>Finally, I had to change the settings of the serial port using <code>stty</code>. +The settings needed to be:</p> +<ul> +<li>9600 baud rate</li> +<li>8 character bits</li> +<li>1 stop bit</li> +<li>No parity</li> +</ul> +<p>So in order to set them accordingly I ran:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>stty -F /dev/ttyS1 <span style="color:#ae81ff">9600</span> cs8 -cstopb -parenb +</span></span></code></pre></div><p>And finally, I sent some data to test the printer.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>echo <span style="color:#e6db74">&#34;This is a test&#34;</span> &gt; /dev/ttyS1 +</span></span></code></pre></div><p>And&hellip;<strong>it worked!</strong></p> + + +<figure ><img src="https://brycev.com/p/testprint.webp" alt="It works"><figcaption>Success!</figcaption></figure> + +<h3 id="printing-pdfs">Printing PDFs</h3> +<p>Printing text from a modern computer onto a 30 year old printer is pretty +cool, but I wanted to see how well it could print PDFs. I actually planned +to write a program to translate a PDF to a format that I could output to +the ImageWriter, needless to say, this would take a while. Luckily, I found +out that <a href="https://www.ghostscript.com/">Ghostscript</a> actually supports output +to the Apple ImageWriter family. Four drivers where available:</p> +<ul> +<li>appledmp (Generic Apple dot matrix driver)</li> +<li>iwlo (Apple ImageWriter)</li> +<li>iwhi (Apple ImageWriter II)</li> +<li>iwlq (Apple ImageWriter LQ)</li> +</ul> +<p>I choose <a href="https://upload.wikimedia.org/wikipedia/commons/a/af/Tux.png">this</a> +as my test image.</p> +<p>In order to create a file that I could send to the printer I needed to +first convert the PNG into a PDF file with ImageMagick, run Ghostscript, +and then output it to the printer:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>convert Tux.png Tux.pdf +</span></span><span style="display:flex;"><span>gs -q -dBATCH -dNOPAUSE -sDEVICE<span style="color:#f92672">=</span>iwhi -sPAPERSIZE<span style="color:#f92672">=</span>letter -dFIXEDMEDIA -dPSFitPage -r160x144 -sOutputFile<span style="color:#f92672">=</span>tux Tux.pdf +</span></span><span style="display:flex;"><span>cat tux &gt; /dev/ttyS1 +</span></span></code></pre></div><p>The output from the printer was&hellip;<em>interesting</em>.</p> + + +<figure ><img src="https://brycev.com/p/tuxtest.webp" alt="Failed print"></figure> + +<blockquote> +<p>Side note, you can see the visible lines that go across the print. That is +actually the broken pin on the print head, I plan on replacing the print +head eventually.</p></blockquote> +<p>This was good and bad. Good because graphics printing works. +Bad because I had no idea what was causing this image to corrupt. +At this point I had no idea what was wrong. I assumed the problem was that +the Ghostscript driver was faulty, however I tried all 4 Apple ImageWriter +drivers and they all had a similar result. At one point I thought the serial +connection was bad, but these errors were too consistent to be a connection +issue. I looked at the +<a href="https://mirrors.apple2.org.za/ftp.apple.asimov.net/documentation/hardware/printers/Apple%20ImageWriter%20II%20Technical%20Reference%20Manual.pdf">Apple ImageWriter II Technical Reference Manual</a> +and that&rsquo;s when I found the problem.</p> +<hr> +<h4 id="a-side-note-about-printers">A Side Note About Printers</h4> +<p>One thing that I was ignorant about when it came to printers is that +almost every printer has a <strong>buffer</strong>. The buffer stores data for the current +print job (sometimes future print jobs as well), feeds it to the printer +once it needs to print that data, and then disposes of that data once it +has been printed. Most modern printers have a decent buffer size to hold +print data. Most printing software keeps track of how much data +is sent to the printer buffer. This means that we never really have to worry +about prints being messed up since print buffers are automatically managed +for us.</p> +<hr> +<p>Once I read that the Apple ImageWriter II only has a print buffer size of +<strong>2 kilobytes</strong> I knew what the problem was.</p> +<p>The problem about sending data at 9600 baud to the printer was that it was +sending data at a faster rate than the printer could actually print, which was +overwriting data in the buffer which caused it to print garbage. In order +to remedy this I did something no man has ever done&hellip;<em>lower</em> the baud rate.</p> +<p>Once I set the baud rate on the printer and my computer&rsquo;s serial port to +2400 baud and printed the image, I got this:</p> + + +<figure ><img src="https://brycev.com/p/matrixtux.webp" alt="Printed image of Tux"></figure> + +<p><strong>Success!</strong></p> +<p>There are a few errors on the right of Tux and a missing line, but that is +just a problem with Ghostscript that I can&rsquo;t fix. It usually does not show +up on most other prints.</p> +<h3 id="printing-faster">Printing Faster</h3> +<p>The printer works perfectly at 2400 baud and lower. +The only problem is that this printer is now <em>slow</em>. 2400 baud is +not fast at all! There are pauses during printing where the printer is +waiting to receive serial data from the computer. So how do I increase the +data transfer rate but also limit the data transferred to 2kb? Well, that&rsquo;s +where a program called <code>split</code> comes in handy. <code>split</code> is a standard POSIX +utility that is available on <strong>ALL</strong> POSIX systems. It takes an input file +a splits it into multiple output files, but it also has a <code>-b</code> option +which allows you to limit the split files to a specific size. In this case, +we can limit the file size to 2kb and send them one at a time at 9600 baud. +All I have to do is run:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>split -b 2k tux +</span></span></code></pre></div><p>And now we have 67 different 2kb files starting with &ldquo;x&rdquo; that we can send one at a time. +In order to print them one at a time I wrote a script that just loops +over these files:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span><span style="color:#75715e">#!/bin/sh +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span> +</span></span><span style="display:flex;"><span>files<span style="color:#f92672">=</span><span style="color:#e6db74">&#34;</span><span style="color:#66d9ef">$(</span>ls -1 x*<span style="color:#66d9ef">)</span><span style="color:#e6db74">&#34;</span> +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">for</span> x in $files; <span style="color:#66d9ef">do</span> +</span></span><span style="display:flex;"><span> cat $x &gt; /dev/ttyS1 +</span></span><span style="display:flex;"><span> sleep <span style="color:#ae81ff">5</span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">done</span> +</span></span></code></pre></div><p>Of course this is <strong>not</strong> an efficient way of printing. There are moments +where the printer is waiting for data to be received, but this is good enough. +In order to achieve close to 100% efficiency, you would need to +calculate how fast the print head and page feed move on the host, but +I&rsquo;m <em>way</em> too lazy to do that.</p> +<h2 id="the-result">The Result</h2> +<p>The result is a printer that is somewhat slow, but gets the job done. All +I need now is to get <a href="https://en.wikipedia.org/wiki/Continuous_stationery">tractor/continuous feed paper</a> so I don&rsquo;t have to load +paper manually. For me, this printer is definitely a good enough solution +until I get a new printer (if this one even fails).<sup id="fnref:3"><a href="#fn:3" class="footnote-ref" role="doc-noteref">3</a></sup></p> +<video width="640" height="480" preload="none" controls> + <source src="https://brycev.com/print.webm" type="video/webm"> +</video> + +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>I think there actually <em>are</em> thermal printers that can print in color +(I have no idea how), but they seem very rare.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>Dot matrix printers <em>do</em> have more downsides like the fact that pins +on the print head can break easily, but since you can replace the print heads +<strong>and</strong> print heads are relatively cheap this is negligible.&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:3"> +<p>I&rsquo;ve been keeping my eye on any used Epson dot matrix printers. ๐Ÿ‘€&#160;<a href="#fnref:3" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + Why I Switched to Alpine + https://brycev.com/blog/why-i-switched-to-alpine/ + Mon, 06 Nov 2023 00:00:00 +0000 + https://brycev.com/blog/why-i-switched-to-alpine/ + + +<figure ><img src="https://brycev.com/p/alpine.webp" alt="Alpine Linux logo"></figure> + +<p>I&rsquo;ve been a <a href="https://voidlinux.org/">Void Linux</a> user for about 2 years and I want to say it has been +a good run. Void Linux has been an absolutely great Linux distribution and +has served me well for quite a while. However, every Linux distribution is not +without it&rsquo;s shortcomings. Although Void Linux is a <strong>great</strong> distribution, +I still think that it had a few shortcomings that <a href="https://alpinelinux.org/">Alpine Linux</a> fixes.</p> +<h2 id="what-alpine-does-better-than-void">What Alpine Does Better Than Void</h2> +<h3 id="packages">Packages</h3> +<p>Although the main Void Linux repositories have more packages than most other +distributions, the selection is still a bit&hellip;lacking. As of writing this, +Void Linux has around 13,369 packages. Which is still <strong>a lot</strong> of packages, +however Alpine Linux more than <strong>doubles</strong> Void at around 28,583 packages<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>.</p> +<p>One thing that bugs me is that even though Void supports musl libc, not all +13,369 packages are available for musl. As you would have guessed, Alpine does +not have this problem since musl is the default.</p> +<p>Another thing that kind of bugs me is how long it took for packages I +submitted to Void&rsquo;s package repos to get accepted. Some of my requests +were <strong>never</strong> responded to and others were just forgotten about entirely. +When submitting packages to Alpine&rsquo;s packages repos my requests were almost +answered the same day and 100% of my packages (so far) have been accepted +into the testing branch.</p> +<h3 id="xbps">XBPS</h3> +<p>Now don&rsquo;t get me wrong, the XBPS package manager is great, however apk +(Alpine&rsquo;s package manager) is superior when compared to XBPS. For example, when updating +my system after a month on Void it would take around 3-4 minutes to update +using XBPS. Whenever I update after a month on Alpine it usually takes +less than a minute!</p> +<p>A set of packages that show this perfectly are the <code>texlive</code> packages. The +time it takes to install an entire texlive distribution on disk is +notoriously long. Alpine&rsquo;s apk installs texlive <em>way</em> faster than XBPS. +Not only that, but apk manages cached packages in a much better way than +XBPS, in my opinion.</p> +<p>I don&rsquo;t think I will ever see a package manager as fast, light, and minimal +as apk.</p> +<h3 id="busybox">BusyBox</h3> +<p>One thing that keeps Alpine small is instead of using the GNU Coreutils, +Alpine uses the BusyBox Coreutils. For a long time user of the GNU Coreutils +like me, the switch can be very jarring, however the payoff is (in my opinion) +worth it. The BusyBox Coreutils are actually easier to learn since they have +less features and options. They also have the upside of being more POSIX +compliant than the GNU Coreutils so writing scripts with BusyBox usually +results in more portability.</p> +<h3 id="musl-musl-musl-">musl, musl, musl ๐Ÿš</h3> +<p>Void and Alpine both support musl libc, but a major difference is that Alpine +supports <strong>only</strong> musl<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup>. For those of you who are not well versed in C +libraries, musl is essentially a smaller and more tidy C library replacement +for glibc (the GNU C library). As I mentioned above, this means that all +28,583 Alpine packages are compiled for musl.</p> +<p>For reasons that I won&rsquo;t get into (since it would take too long) musl is +generally better than glibc, but sometimes isn&rsquo;t as compatible as glibc, +you can find out more about that <a href="https://wiki.musl-libc.org/functional-differences-from-glibc.html">here</a>. +I think one of the best comparisons of musl and glibc can be found +<a href="https://www.etalabs.net/compare_libcs.html">here</a>. musl binaries tend to +be smaller, faster, and more portable compared to glibc.</p> +<p>As a C programmer, working with musl is a <strong>dream</strong> compared to glibc.</p> +<h3 id="its-great-use-it-already">It&rsquo;s Great, Use It Already</h3> +<p>Overall, Alpine Linux fixes almost every gripe that I had with Void Linux +and also fixes problems that I didn&rsquo;t even know I had. The entire Alpine +Linux distribution is easy to understand, simple, and fast. +There are many, many +more things that Alpine does better than Void, but if I listed them <em>all</em> +then it would take forever. But so far, Alpine Linux is probably one of the +<strong>best</strong> Linux distributions that I have ever used.</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>These numbers where obtained by running <code>xbps-query -Rs &quot;*&quot; | wc -l</code> +and <code>apk search &quot;*&quot; | wc -l</code> respectively.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>Technically Alpine <em>does</em> support glibc if you wish to install it +through apk.&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + FFmpeg Video Calls + https://brycev.com/blog/ffmpeg-video-calls/ + Mon, 18 Sep 2023 00:00:00 +0000 + https://brycev.com/blog/ffmpeg-video-calls/ + <p>If you have seen any of the <a href="https://youtu.be/VGRHzB8ANAo">videos I made on</a> +<a href="https://youtu.be/4NKmEjzfJ98">FFmpeg</a> you would know that I absolutely love +it and I think that it is the absolute <strong>BEST</strong> video/multimedia program ever +made.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup> FFmpeg has so many features and it implements almost all of them +extremely well. So it comes to no surprise, after I initially made my FFmpeg +videos I found out that FFmpeg can actually be used to create video streams and +send them over the internet. After I found out about this I wondered if it +could be used as a sort of minimal way to create video calls. After +experimenting for just a few <em>minutes</em> I found a way to stream video over the +internet via FFmpeg.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>ffmpeg -f alsa -i default <span style="color:#ae81ff">\ </span><span style="color:#75715e"># Get audio from mic</span> +</span></span><span style="display:flex;"><span>-i /dev/video0 <span style="color:#ae81ff">\ </span><span style="color:#75715e"># Get video from camera</span> +</span></span><span style="display:flex;"><span>-s 640x480 <span style="color:#ae81ff">\ </span><span style="color:#75715e"># Scale down to 640x480</span> +</span></span><span style="display:flex;"><span>-c:v libx264 -preset:v ultrafast <span style="color:#ae81ff">\ </span><span style="color:#75715e"># Encode fast!</span> +</span></span><span style="display:flex;"><span>-tune zerolatency -intra-refresh <span style="color:#ae81ff">1</span> <span style="color:#ae81ff">\ </span><span style="color:#75715e"># Reduce latency</span> +</span></span><span style="display:flex;"><span>-f mpegts <span style="color:#ae81ff">\ </span><span style="color:#75715e"># Put everything into mpegts stream</span> +</span></span><span style="display:flex;"><span>-b:v 1M udp://localhost:1313 <span style="color:#75715e"># Send video over localhost:1313</span> +</span></span></code></pre></div><p>In order to actually view this video you would need a video player that is +able to read/receive from UDP. Luckily such a program exists: +<a href="https://youtu.be/iR76e9XUodI">mpv</a>.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>mpv --profile<span style="color:#f92672">=</span>low-latency udp://localhost:1313 +</span></span></code></pre></div><p>Now to actually send this over the internet and not just over LAN or localhost you +would need to replace <code>localhost:1313</code> with the IP address of the recipient +for FFmpeg and the IP address of the sender for mpv. This is will let +you be able to do peer-to-peer video calls using only FFmpeg and mpv (or any +other UDP compatible video player). FFplay, a media player that comes with +FFmpeg, can play UDP streams, so technically this can be done using <strong>only</strong> +FFmpeg. If you replace <code>-f /dev/video0</code> with +<code>-i x11grab</code> you can also screencast to another computer as well, pretty neat.</p> +<p>Theoretically, you could do more than just one-on-one video calls, you could +have 4, 7, or even 15 people using this method. It would definitely be tedious +to set it up, but it would be possible. And of course if you don&rsquo;t want to use +video at all you can easily use this method for audio calls.</p> +<p>I think this example shows just how versatile FFmpeg can be. Not only that +but the settings I used for this example are not actually optimal, it was +just a test but it worked great. This is just the tip of the iceberg too, +you can also use this to stream movies from another computer, +video games, general live streaming, and much more. +I personally +find it crazy that you can basically do audio and video calls using <strong>only</strong> +FFmpeg and this shows just how absolutely amazing it is.</p> +<video width="640" height="480" preload="none" controls> + <source src="https://brycev.com/cameracast.webm" type="video/webm"> +</video> + +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>In fact, the creator of FFmpeg, Fabrice Bellard, also made QEMU and the +Tiny C Compiler (TCC) both of which are very widely used pieces of software +that are very well made.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + A Branded Life + https://brycev.com/blog/a-branded-life/ + Thu, 06 Jul 2023 00:00:00 +0000 + https://brycev.com/blog/a-branded-life/ + <p>Modern &ldquo;brands&rdquo; are a weird thing when you really think about them. +Disney, Hewlett Packard, and Nike. What do all 3 of these have in common? +They&rsquo;re all considered brands. But what exactly is a brand? Before the +industrial age, a brand was just a way to differentiate a product from a +competitor&rsquo;s product. In order to differentiate it one would often <em>brand</em> +their name or alias in the product itself.<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup> However, the concept of a +brand has evolved over the last 100-200 years. Instead of being just a +name or logo, a brand is often associated with what people hear +(jingles/slogans), taste, see, and more. How many times have you heard +someone say, &ldquo;Hey, this <strong>X</strong> is just like <strong>Y</strong> brand?&rdquo; As a result of this +recognition, modern branding strategies have exploded in popularity. For the +companies that brand their products like this, it creates more customer +loyalty and therefore more money.</p> +<p>But is branding (in the modern sense) a good thing?</p> +<h2 id="everything-is-a-brand">Everything is a Brand</h2> +<p>One thing that I tend to find very annoying is the modern obsession of brands +to the point where <strong>everything</strong> has to be a brand. From homemade crafts to +personal websites, it seems like there are so many attempts at creating a +modern brand that are just not necessary.</p> +<p>Let me share an experience that I had somewhat recently. I had a friend +buy me a book from a local book shop as a gift. When I received it I thanked +them for the gift and I read it later. I noticed something rather odd +about this book&hellip;there was <strong>no</strong> branding. <strong>No</strong> <em>brand name</em>, <strong>no</strong> name, and +not even a <strong>copyright notice</strong>. Needless to say I was taken back by this +as it seemed abnormal. But why did I think it was abnormal? If I had to guess, +it was probably because seeing someone not milking their work and +committing a sort of selfless act seemed out of place, especially in this +day and age.</p> +<p>When was the last time you saw a book without a logo or brand name? +I only have two books in my <em>entire</em> collection that fit that criteria. +Actually, when is the last time you saw <strong>anything</strong> without a logo +or brand name? Go through the room you&rsquo;re in right now and count how many +items you have that don&rsquo;t have a brand name and or logo. No, seriously, do it. +You will be genuinely surprised.<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup></p> +<h2 id="the-safe-zone">The Safe Zone</h2> +<p>One very important downside of modern brands that impact customers is +what I like to call the &ldquo;<em>safe zone</em>&rdquo;. The &ldquo;safe zone&rdquo; is a situation in which a +customer buys products of specific brands either due to trust, loyalty, +familiarity, or a mix of all three. This results in the customer not buying +anything else unless it&rsquo;s from one of their chosen brands.</p> +<p>This usually results in dependence on a specific brand. This is, of course, +intended. And while it is <strong>very</strong> beneficial to the brand itself, it can +be detrimental to the customer. On one hand, the customer is provided safety +and security knowing that what they get from a specific brand will be +(mostly) consistent. On the other hand, this can trap the customer into +dependence on a brand since not buying from a trusted brand would +be too &ldquo;risky&rdquo; or &ldquo;unsafe&rdquo; (hence the name &ldquo;safe zone&rdquo;).</p> +<p>This effect can happen on such a large scale that many brand names are +immediately associated with a non-branded objects. For example, do you hear +people say cola or Coca-Cola more? Do you hear people say tissue or +Kleenex more? Do you hear people say slow cooker or Crock-Pot more? The +list goes on and on. Not only does this result in other products being +noticed less, but it also draws a negative light on other products and make +them seem inferior compared to larger brands.</p> + + +<figure ><img src="https://brycev.com/p/consoom.webp" title="Must consooooome!" alt="consoomer"></figure> + +<h2 id="drm">DRM</h2> +<p><a href="https://www.defectivebydesign.org/what_is_drm">Digital Restrictions Management</a>, +more commonly referred to as Digital Rights Management, is one way that a +brand keeps customers. DRM essentially restricts what the customer/user +can do with a product. This is usually very prevalent among popular +companies/brands especially in the technology sector.</p> +<p>The best example I can probably think of is ink cartridges for inkjet printers. +Most major brands of inkjet printers use proprietary ink cartridges with DRM +built into the cartridges themselves. If you try to use a different type of ink +cartridge on your printer, <strong>it won&rsquo;t work</strong>. If you try to refill the ink in +your cartridge, <strong>it won&rsquo;t work</strong>. If you buy third party ink cartridges, +<strong>they won&rsquo;t work</strong>.<sup id="fnref:3"><a href="#fn:3" class="footnote-ref" role="doc-noteref">3</a></sup> It&rsquo;s not because it is &ldquo;incompatible&rdquo;, it&rsquo;s ink, +it&rsquo;s because the manufacturer wants to force you into brand loyalty and to +buy their products.</p> +<p>Now it doesn&rsquo;t take a genius to understand that DRM is bad, but for the +handful of people who don&rsquo;t understand why DRM is <em>objectively</em> awful, +here are a few <a href="https://creativecommons.org/2017/07/09/terrible-horrible-no-good-bad-drm/">links to</a> +<a href="https://www.defectivebydesign.org/so_youve_got_some_questions_do_you#examples">understand</a> +<a href="https://www.audioholics.com/news/drm-bad-killing-online-music">why</a>.</p> + + +<figure ><img src="https://brycev.com/p/printer.webp" alt="Printer DRM"><figcaption>Classic printer DRM at work.</figcaption></figure> + +<p>Although the modern conception of branding is very beneficial from an economic +and financial standpoint, it can have many negative side effects outside the +economic and financial realm. It can result in enforcing restrictions on the +customer, physiologically locking them into dependence, and creates obsession +with branding. Although this is great for the brand owners, it sucks for the +everyday customer. Granted, there are many more factors that go into branding +but I think these points really highlight the nature of the current situation.</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>This, of course, is a centuries old tradition and goes all the way back +to branding livestock to prove ownership.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>Unless you make it a habit to tear off labels and branding of products +that you buy. In which case, I wouldn&rsquo;t blame you for doing that.&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:3"> +<p>And this isn&rsquo;t made any better by the fact that inkjet printer ink +is extremely expensive, especially if it is purchased from the manufacturers +themselves.&#160;<a href="#fnref:3" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + Stop Saying C/C++ + https://brycev.com/blog/stop-saying-c-and-c++/ + Thu, 18 May 2023 00:00:00 +0000 + https://brycev.com/blog/stop-saying-c-and-c++/ + <p>For as long as I can remember, I have heard people say C/C++ when referring to +a project written in C and or C++. A lot of programming/developer jobs also +refer to C/C++ when they need a programmer who knows either C or C++. To most +people who have never touched C or C++ this might not seem like a big deal. +However, the problem is that when people say this term (C/C++) they make it +seem like C and C++ are similar or closely related programming languages. +<strong>That is not true.</strong> Although C++ was based off of C when it was first +created, these two languages have slowly drifted apart over the years to the +point where they share less and less in common<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>.</p> +<h2 id="c-and-c-are-very-different">C and C++ are VERY Different</h2> +<p>There is probably someone who is going to say, &ldquo;Well you can write C code in +a C++ program, so technically C is a subset of C++.&rdquo; The only problem is that +you can write C code in <a href="https://ziglang.org/documentation/master/#C">Zig</a>, +<a href="https://pkg.go.dev/cmd/cgo">Go</a>, <a href="https://github.com/nim-lang/Nim/wiki/Nim-for-C-programmers">Nim</a>, +and basically almost every other language out there has a C FFI! So should I refer to Zig, +Go, and Nim as C/Zig, C/Go, and C/Nim? Obviously <strong>no</strong>.</p> +<h3 id="c-with-classes">C with Classes</h3> +<blockquote> +<p>&ldquo;But C++ is just C with classes!&rdquo;</p></blockquote> +<p><strong>No</strong>, it isn&rsquo;t. Anyone who says this obviously has never worked with C++. C++ has +completely different standard libraries, implementations, and standards than C. +It is true that when C++ was first made it was just C with classes<sup id="fnref:2"><a href="#fn:2" class="footnote-ref" role="doc-noteref">2</a></sup>, that has +long been false ever since C++ has implemented features separate from C.</p> +<h3 id="incompatibility">Incompatibility</h3> +<h4 id="void-pointers">Void Pointers</h4> +<p>One such case where C++ is incompatible with C is with void pointers. +For example, this program will compile with a C compiler (like GCC), but it +will not compile with a C++ compiler (like G++):</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-c" data-lang="c"><span style="display:flex;"><span><span style="color:#75715e">#include</span> <span style="color:#75715e">&lt;stdlib.h&gt;</span><span style="color:#75715e"> +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">int</span> <span style="color:#a6e22e">main</span>() { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">int</span> <span style="color:#f92672">*</span>a <span style="color:#f92672">=</span> <span style="color:#a6e22e">malloc</span>(<span style="color:#ae81ff">5</span>); +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> <span style="color:#ae81ff">0</span>; +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>All this code does is allocate 5 bytes to an integer pointer <code>a</code>. This program +works perfectly fine when compiled with GCC, but if I compile this program with +G++ this error is returned:</p> +<pre tabindex="0"><code>main.c: In function &#39;int main()&#39;: +main.c:4:24: error: invalid conversion from &#39;void*&#39; to &#39;int*&#39; [-fpermissive] + 4 | int *a = malloc(5); + | ~~~~~~^~~ + | | + | void* +</code></pre><p>The reason this happens is that <code>malloc</code> returns a void pointer and C++ cannot +convert a void pointer into an integer pointer unless it is specifically cast +to an integer pointer.</p> +<h4 id="kr-syntax">K&amp;R Syntax</h4> +<p>Another big incompatibility with C and C++ is that C++ is actually incompatible +with <a href="https://en.wikipedia.org/wiki/C_(programming_language)#K&amp;R_C">K&amp;R</a> syntax. Given this example function formatted in K&amp;R syntax:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-c" data-lang="c"><span style="display:flex;"><span><span style="color:#66d9ef">int</span> <span style="color:#a6e22e">gcd</span>(a, b) +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">int</span> a; +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">int</span> b; +</span></span><span style="display:flex;"><span>{ +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">if</span> (b <span style="color:#f92672">==</span> <span style="color:#ae81ff">0</span>) +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> a; +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> <span style="color:#a6e22e">gcd</span>(b, (a <span style="color:#f92672">%</span> b)); +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>It will compile perfectly fine for GCC (as expected), however G++ gives us +<em>another</em> set of errors&hellip;</p> +<pre tabindex="0"><code>gcd.c:3:9: error: &#39;a&#39; was not declared in this scope + 3 | int gcd(a, b) + | ^ +gcd.c:3:12: error: &#39;b&#39; was not declared in this scope + 3 | int gcd(a, b) + | ^ +gcd.c:3:13: error: expression list treated as compound expression in initializer [-fpermissive] + 3 | int gcd(a, b) + | ^ +gcd.c:6:1: error: expected unqualified-id before &#39;{&#39; token + 6 | { + | ^ +</code></pre><p>This makes it almost impossible to use K&amp;R syntax with C++ unless you format +your function arguments according to <a href="https://gist.github.com/nicholatian/2d9514feaf9a95e7561a433ac404b141">ANSI C</a>. +(I know not many people care about K&amp;R syntax, but I think that it is still an +important difference).</p> +<p>There are also many other things in C that will not transfer over to C++ like +complex numbers, default return types, and more, but +I think you already get the picture by now. These incompatibilities are not +anything that would break the entire C language if used in conjunction with +C++, but these small differences slowly add up.</p> +<h3 id="hard-for-beginners">Hard for Beginners</h3> +<p>Not differentiating between C and C++ also has the side effect of ostracizing new +users. Many beginner programmers are lead by the term &ldquo;C/C++&rdquo; to think that +they&rsquo;re basically the same language. In fact there are <a href="https://medium.com/@yekayama/stop-making-c-c-tutorials-2fa9bc114488">many</a> tutorials out there +that are advertised as &ldquo;C/C++ tutorials&rdquo;, continuing the confusion. +This can also scare away C beginners by making them think that understanding +the complexities of C++ are required to understand C (<em>SPOILER</em>: They&rsquo;re not). +I have fallen for this trap in the past, as well as many others. +C is honestly a very simple programming language, C++ is not.</p> +<h2 id="c-and-c-programmers-are-very-different">C and C++ Programmers are VERY Different</h2> +<p>With the new C++ standards given throughout the years like C++11, C++20, and +etc. C++ programmers have been given more tools and functions that don&rsquo;t exist +in standard C. This usually results in modern C programs having more lines of +code than modern C++, however this means that modern C is usually more readable +than modern C++. Here is an example question from <a href="https://leetcode.com/problems/maximum-count-of-positive-integer-and-negative-integer/">LeetCode</a>. +Solutions differ, but most C solutions look something like this:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-c" data-lang="c"><span style="display:flex;"><span><span style="color:#66d9ef">int</span> <span style="color:#a6e22e">maximumCount</span>(<span style="color:#66d9ef">int</span> <span style="color:#f92672">*</span>nums, <span style="color:#66d9ef">int</span> numsSize) { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">int</span> pos <span style="color:#f92672">=</span> <span style="color:#ae81ff">0</span>, neg <span style="color:#f92672">=</span> <span style="color:#ae81ff">0</span>; +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">for</span> (<span style="color:#66d9ef">int</span> i <span style="color:#f92672">=</span> <span style="color:#ae81ff">0</span>; i <span style="color:#f92672">&lt;</span> numsSize; i<span style="color:#f92672">++</span>) { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">if</span> (nums[i] <span style="color:#f92672">&gt;</span> <span style="color:#ae81ff">0</span>) pos<span style="color:#f92672">++</span>; +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">else</span> <span style="color:#66d9ef">if</span> (nums[i] <span style="color:#f92672">&lt;</span> <span style="color:#ae81ff">0</span>) neg <span style="color:#f92672">++</span>; +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> pos <span style="color:#f92672">&gt;</span> neg <span style="color:#f92672">?</span> pos : neg; +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>Even though this code is pretty compact for C standards it is still <em>very</em> +readable. Now for the C++ solutions, there are a lot of variations to this +solution so I will use one that&rsquo;s different <em>enough</em> from C.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-cpp" data-lang="cpp"><span style="display:flex;"><span><span style="color:#66d9ef">int</span> <span style="color:#a6e22e">maximumCount</span>(std<span style="color:#f92672">::</span>vector<span style="color:#f92672">&lt;</span><span style="color:#66d9ef">int</span><span style="color:#f92672">&gt;</span> nums) { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">auto</span> [a, b] <span style="color:#f92672">=</span> std<span style="color:#f92672">::</span>equal_range(nums.begin(), nums.end(), <span style="color:#ae81ff">0</span>); +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> std<span style="color:#f92672">::</span>max(std<span style="color:#f92672">::</span>distance(nums.begin(), a), std<span style="color:#f92672">::</span>distance(b, nums.end())); +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>This uses <code>vector</code> and <code>algorithm</code> from the C++ standard library<sup id="fnref:3"><a href="#fn:3" class="footnote-ref" role="doc-noteref">3</a></sup>. +As you can see this code is <em>much</em> more compact but is definitely not as +readable as the C code. Although the C solution can be compiled by a C++ +compiler I wanted to highlight just how different they can be from each other. +This is but one example of how C and C++ programmers have slowly separated +when it comes to programming.</p> +<h3 id="many-c-programmers-wont-touch-c">Many C Programmers Won&rsquo;t Touch C++</h3> +<p>I&rsquo;m pretty sure everyone knows the C programmer stereotype by now, the only +thing is that it is <strong>TRUE</strong>. +Lots of <a href="https://suckless.org/">Suckless</a> users and developers only use +C and POSIX shell in their programs. <a href="https://harmful.cat-v.org/software/c++/">Cat-v</a> endorses +C and C-like languages, but despises C++. Even +<a href="https://lore.kernel.org/all/alpine.LFD.0.999.0709061839510.5626@evo.linux-foundation.org/">Linus Torvalds</a>, +the creator Linux and Git, won&rsquo;t touch C++. +Heck, even I love C but I can&rsquo;t stand programming in C++.</p> +<p>This is probably the biggest reason why employers <strong>SHOULD NOT</strong> put C/C++ +on job descriptions, especially if they&rsquo;re only looking for C developers. +All they are doing is scaring away competent C developers.</p> + + +<figure ><a href="https://brycev.com/p/cpp.webp"><img src="https://brycev.com/p/cpp.webp" id="smallimg" alt="Stop doing C&#43;&#43;!"></a></figure> + +<h2 id="the-solution">The Solution</h2> +<p>If you&rsquo;re referring to a C program or programmer just say &ldquo;C&rdquo;. +If you&rsquo;re referring to a C++ program or programmer just say &ldquo;C++&rdquo;. +If you&rsquo;re referring to both used separately say something like:</p> +<ul> +<li>C and C++</li> +<li>C, C++</li> +<li>C++ with C</li> +<li>Etc.</li> +</ul> +<p><strong>NOT C/C++</strong></p> +<p>Only if you&rsquo;re using C <em>together</em> with C++ would it be acceptable to +say C/C++.</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>Please note that although these are real concerns with C and C++, +this is more of a rant than anything else (and somewhat satire).&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:2"> +<p>Fun fact: C++ <em>was</em> actually called <a href="https://www.stroustrup.com/bs_faq.html#invention">&ldquo;C with Classes&rdquo;</a> before it was +initially released.&#160;<a href="#fnref:2" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +<li id="fn:3"> +<p>Credit to <a href="https://youtu.be/U6I-Kwj-AvY">code_report</a> for these two solutions.&#160;<a href="#fnref:3" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + I Finally Have a VPS + https://brycev.com/blog/i-finally-have-a-vps/ + Sun, 12 Mar 2023 00:00:00 +0000 + https://brycev.com/blog/i-finally-have-a-vps/ + <p>After a little less than a year of waiting, I finally obtained a VPS +from <a href="https://my.frantech.ca/">Frantech/BuyVM</a>. I was told that they are an +exceptional VPS provider, and it turns out to be <strong>true</strong>. Their pricing is +extremely fair as I can get <em>double</em> what <a href="https://www.vultr.com/">Vultr</a> +offers for the same exact price (plus unlimited bandwidth). Their support +was also pretty fast so I was able to setup everything relatively quickly. +My only complaint is that the number of servers is limited so it might take +a while to actually get one. Despite Frantech <em>obviously</em> being better, Vultr +is still a solid choice for a VPS and they make it very easy to setup one.</p> +<p>I am now hosting my web page from my VPS instead of <a href="https://sourcehut.org/">Sourcehut</a> +and I am hosting my email instead of using <a href="https://disroot.org">Disroot</a>. +Keep in mind that these services (Sourcehut and Disroot<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>) are great services +and I would 100% recommend them, however I believe that hosting everything +yourself is better.</p> +<p>That being said, I do have unlimited bandwidth and about 19GB of storage space +that I still have yet to use. If any of you guys have ideas for what I should +do (hosting internet radio, SearX instance, Mastodon instance, forum, etc.) you +can let me know by email.</p> +<p>If you would like a VPS for yourself, you can use the links above to get one +from Frantech, if they&rsquo;re available. If not, Vultr is always a solid option +for a VPS (although you do get less). <strong>Or</strong> you can use my referral links +to get a VPS below. It&rsquo;s a nice and indirect way to support me.</p> +<h3 id="referral-links">Referral Links</h3> +<ul> +<li><a href="https://my.frantech.ca/aff.php?aff=6418">Frantech/BuyVM</a></li> +<li><a href="https://www.vultr.com/?ref=9386356">Vultr</a> (Normal affiliate link)</li> +<li><a href="https://www.vultr.com/?ref=9386357-8H">Vultr</a> (<strong>Get $100</strong> to use for a VPS)</li> +</ul> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>Disroot provides services like cloud storage, XMPP chat, paste bin, Git, +and more. They&rsquo;re an amazing replacement for the Google suite.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + Odysee Channel + https://brycev.com/blog/odysee-channel/ + Wed, 08 Mar 2023 00:00:00 +0000 + https://brycev.com/blog/odysee-channel/ + <p>For those of you who don&rsquo;t already know, I recently synced my entire YouTube +channel to <a href="https://odysee.com/@bryce:c">Odysee</a>. So for those of you who want to avoid YouTube and Google +entirely that is now an option. I will continue to upload videos to YouTube +but they will be automaticly synced to Odysee shortly after I upload them +(usually takes about an hour). I don&rsquo;t plan on uploading exclusively onto +Odysee, but I might post some videos that I normally wouldn&rsquo;t post onto +YouTube for one reason or another, so keep an eye out. ๐Ÿง‘โ€๐Ÿš€</p> + + + + + Updating My GPG Key + https://brycev.com/blog/updating-my-gpg-key/ + Sun, 15 Jan 2023 00:00:00 +0000 + https://brycev.com/blog/updating-my-gpg-key/ + <p>I have updated my GPG key. I will <strong>not</strong> be able to read any new encrypted emails +that use my old GPG key.</p> +<p>You can find my new GPG key <a href="https://brycevandegrift.xyz/bpv.gpg">here</a> or import it +directly by running:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>curl -fL <span style="color:#e6db74">&#34;https://brycevandegrift.xyz/bpv.gpg&#34;</span> | gpg --import +</span></span></code></pre></div> + + + + Why I Am Switching to Firefox + https://brycev.com/blog/why-i-am-switching-to-firefox/ + Thu, 05 Jan 2023 00:00:00 +0000 + https://brycev.com/blog/why-i-am-switching-to-firefox/ + <p>For the past year I have used <a href="https://qutebrowser.org/">Qutebrowser</a> almost exclusively as my main +web browser. I even made a video on how extensible and great it is <a href="https://youtu.be/N79au5Xq65s">here</a>. +However there have been a few glaring issues with it over other browsers +like Firefox that I have noticed over the past year. Although Qutebrowser +is a great web browser, I think that these issues are too important for +me to ignore.</p> +<p>Now before I start bashing Qutebrowser on where it falls short of my +expectations, I should rightfully praise it for what it does right compared +to other web browsers. Qutebrowser does have a lot of cool and innovative +features that most web browsers wouldn&rsquo;t even think about implementing.</p> +<h2 id="what-qutebrowser-does-right">What Qutebrowser does right</h2> +<h3 id="keybindings-and-ui">Keybindings and UI</h3> +<p>Unlike most browsers, Qutebrowser has Vim keybindings by default. This means +that every single action is bound to a specific key or combination of keys. +This makes using and navigating in Qutebrower leagues faster than in other +browsers like Firefox. There are extensions for Firefox that do allow you +to use Vim keybindings, however they are not as tightly integrated into the +browser as Qutebrowser. For Qutebrowser, <strong>every single action has a keybinding</strong>.</p> +<p>The user interface for Qutebrowser (or possibly lake thereof), in my opinion, +is very clean and nice to look at. Not only does it look nice, but it is also +highly customizable so you can change it to suit your personal preference.</p> +<h3 id="adblocking-by-default">AdBlocking by default</h3> +<p>By default, Qutebrowser has simple hosts adblocking enabled. Even though +there are far better adblocking solutions (uBlock Origin is the gold standard), +Qutebrowser still earns some points for having some sort of adblocking enabled +by default. Qutebrowser also has an adblocking mode simular to Brave Browser&rsquo;s +adblocking mode if hosts adblocking isn&rsquo;t enough for you.</p> +<h3 id="python">Python</h3> + + +<figure ><a href="https://brycev.com/p/boomer-python.webp"><img src="https://brycev.com/p/boomer-python.webp" alt="Boomer stuff"></a></figure> + +<p>Now I know that using Python as a language for writing a big application has +some serious drawbacks, however in the case of Qutebrowser, writing it in +Python gives it some unique traits. For starters, writing Qutebrowser in +Python makes it easier to customize and extend in numerous ways. Being able +to program in my own keybindings, extra features, and more is (for a lack of a +better word) <strong>awesome</strong>. Although there are better language choices than +Python (for example Lua is a solid choice<sup id="fnref:1"><a href="#fn:1" class="footnote-ref" role="doc-noteref">1</a></sup>) for what it is, having +Qutebrowser written in Python comes with some great benefits</p> +<h2 id="what-qutebrowser-does-wrong">What Qutebrowser does wrong</h2> +<h3 id="bloated">Bloated</h3> +<p>This point somewhat ties into the fact that Qutebrowser is written in Python, +but Qutebrowser is anything but small or fast. Qutebrowser uses just about +the same amount of resources on my computer as Firefox (which is considered a +bloated browser) and still runs slower. It also takes up a lot more space +on my system than Firefox as well. The executable for Qutebrowser isn&rsquo;t +actually that big, however the dependices needed to run it take up more space +than Firefox itself.</p> +<h3 id="adblocking">Adblocking</h3> +<p>Like I stated earlier, Qutebrowser has adblocking enabled by default, which is +really nice, however I don&rsquo;t think that its builtin adblocker does enough. +My standard for adblocking (as mentoned before) is uBlock Origin, and the +adblocking options that Qutebrowser provides are not really too impressive. +Compared to uBlock Origin, Qutebrowser&rsquo;s adblockers don&rsquo;t nearly do as much.</p> +<h3 id="python-1">Python</h3> +<p>Didn&rsquo;t I just say that writing Qutebrowser in Python was a good thing? +Well&hellip;yes, but actually no. Python does provide Qutebrowser with an easy +language to write the program in as well as configure and extend the program +in, but the biggest downside of writing a browser in Python is speed. Qutebrowser, +being written in Python, is just a little less resource intensive than Firefox +on my system, however it is about half as fast. This compromise in performance is +expected for a program written in Python and for a browser, performance is +pretty important.</p> +<h2 id="why-firefox">Why Firefox?</h2> +<p>Firefox is the only web browser that is free software and also mitigates +all of the problems I have discussed. Even though Firefox is still a bit +bloated, it does fix adblocking (with uBlock Origin) and it is written in a more sane programming +language. Firefox does not come with adblocking or keybindings by default like +Qutebrowser, but they can be added in with extensions like the aforementioned +<a href="https://ublockorigin.com/">uBlock Origin</a> or <a href="https://tridactyl.xyz/">Tridactyl</a>. +It is sometimes a pain to change the default settings for Firefox since a lot +of useless and harmful things are enabled by default like telemetry, Pocket, +Firefox Accounts, and more. After all of that is said and done, Firefox is +actually a pretty good web browser.</p> +<p>Although Firefox is bloated, sometimes you need a bloated browser for +the bloated internet.</p> +<div class="footnotes" role="doc-endnotes"> +<hr> +<ol> +<li id="fn:1"> +<p>There is actually a web browser written in Lua called <a href="https://luakit.github.io/">LuaKit</a>.&#160;<a href="#fnref:1" class="footnote-backref" role="doc-backlink">&#x21a9;&#xfe0e;</a></p> +</li> +</ol> +</div> + + + + + Rust in the Linux Kernel + https://brycev.com/blog/rust-in-the-linux-kernel/ + Tue, 25 Oct 2022 00:00:00 +0000 + https://brycev.com/blog/rust-in-the-linux-kernel/ + <p>I know, I know, everyone is talking about support for the <a href="https://www.rust-lang.org/">Rust</a> programming +language being <a href="https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=8aebac82933ff1a7c8eede18cab11e1115e2062b">added to the Linux kernel</a>, and I&rsquo;m no exception. This event +is <strong>HUGE</strong> when it comes to Linux kernel development. Since the Linux kernel +is predominantly written in C and has been that way for about the last 30 +years, this comes as a very big surprise. I honestly would have expected +C++ to be integrated in the Linux kernel if it weren&rsquo;t for Linus Torvald&rsquo;s +<a href="https://lore.kernel.org/all/alpine.LFD.0.999.0709061839510.5626@evo.linux-foundation.org/">hatred for C++</a> (granted, I dislike C++ as well).</p> +<p>Now I&rsquo;m not here to praise Rust as some sort of gift to the Linux kernel, nor +am I here to talk horribly about it and say that it has no use in the Linux +kernel whatsoever, because it does have a use. I am here to take a look at +what Rust does and doesn&rsquo;t do good and see if that lines up with the needs of +the Linux kernel.</p> +<h2 id="speedperformance">Speed/Performance</h2> +<p>The average speed of programs written in Rust is about on par with programs +written in C from what I&rsquo;ve seen. You can look at some of the benchmarks +<a href="https://programming-language-benchmarks.vercel.app/c-vs-rust">here</a>, but I also have a quick and simple one that I whipped up. +This example uses a very lazy and inefficient version of the Fibonacci +algorithm that I used a while ago in my <a href="https://brycevandegrift.xyz/blog/the-importance-of-lisp/#recursion">lisp article</a>.</p> +<p>Now I&rsquo;m no expert programmer so don&rsquo;t complain if I didn&rsquo;t write either of +these programs &ldquo;correctly&rdquo;, it&rsquo;s just a simple benchmark</p> +<p><a href="https://brycevandegrift.xyz/hardware/">Machine specs</a></p> +<p>Implemented in C:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-c" data-lang="c"><span style="display:flex;"><span><span style="color:#75715e">#include</span> <span style="color:#75715e">&lt;stdio.h&gt;</span><span style="color:#75715e"> +</span></span></span><span style="display:flex;"><span><span style="color:#75715e">#include</span> <span style="color:#75715e">&lt;stdlib.h&gt;</span><span style="color:#75715e"> +</span></span></span><span style="display:flex;"><span><span style="color:#75715e"></span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">int32_t</span> <span style="color:#a6e22e">fib</span>(<span style="color:#66d9ef">int32_t</span> num) { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">if</span> (num <span style="color:#f92672">&lt;=</span> <span style="color:#ae81ff">1</span>) { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> num; +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> <span style="color:#a6e22e">fib</span>(num <span style="color:#f92672">-</span> <span style="color:#ae81ff">1</span>) <span style="color:#f92672">+</span> <span style="color:#a6e22e">fib</span>(num <span style="color:#f92672">-</span> <span style="color:#ae81ff">2</span>); +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">int</span> <span style="color:#a6e22e">main</span>() { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">int32_t</span> result <span style="color:#f92672">=</span> <span style="color:#a6e22e">fib</span>(<span style="color:#ae81ff">45</span>); +</span></span><span style="display:flex;"><span> <span style="color:#a6e22e">printf</span>(<span style="color:#e6db74">&#34;%d</span><span style="color:#ae81ff">\n</span><span style="color:#e6db74">&#34;</span>, result); +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> <span style="color:#ae81ff">0</span>; +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>Implemented in Rust:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-rust" data-lang="rust"><span style="display:flex;"><span><span style="color:#66d9ef">fn</span> <span style="color:#a6e22e">fib</span>(num: <span style="color:#66d9ef">i32</span>) -&gt; <span style="color:#66d9ef">i32</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">if</span> num <span style="color:#f92672">&lt;=</span> <span style="color:#ae81ff">1</span> { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> num; +</span></span><span style="display:flex;"><span> } +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">return</span> fib(num <span style="color:#f92672">-</span> <span style="color:#ae81ff">1</span>) <span style="color:#f92672">+</span> fib(num <span style="color:#f92672">-</span> <span style="color:#ae81ff">2</span>); +</span></span><span style="display:flex;"><span>} +</span></span><span style="display:flex;"><span> +</span></span><span style="display:flex;"><span><span style="color:#66d9ef">fn</span> <span style="color:#a6e22e">main</span>() { +</span></span><span style="display:flex;"><span> <span style="color:#66d9ef">let</span> result:<span style="color:#66d9ef">i32</span> <span style="color:#f92672">=</span> fib(<span style="color:#ae81ff">45</span>); +</span></span><span style="display:flex;"><span> <span style="color:#a6e22e">println!</span>(<span style="color:#e6db74">&#34;</span><span style="color:#e6db74">{}</span><span style="color:#e6db74">&#34;</span>, result); +</span></span><span style="display:flex;"><span>} +</span></span></code></pre></div><p>The average time for C compiled with GCC and <code>-O2</code> optimization was 3.30 +seconds and Rust compiled with RustC and <code>-O2</code> as well was 4.32 seconds. +C without optimizations was 9.31 seconds and Rust without optimizations was +12.69 seconds on average. So unoptimized Rust is a decent bit slower than C +and optimized Rust is about a second behind C which is what I would expect. +A bit of a performance hit is what I would expect for a language that focuses +more on memory safety rather than pure speed.</p> +<h2 id="size">Size</h2> +<p>When it comes to size, Rust and C are both around the same as well. Although +I sometimes do hear complaints about Rust&rsquo;s standard library being large, the +Rust standard library probably isn&rsquo;t going to be included in the Linux kernel. +Compiling a &ldquo;Hello, World!&rdquo; program in GCC with <code>-O2</code> is around 21KB normally +and 15KB stripped. Compiling the same program in RustC with <code>-O2</code> results in a +21KB binary normally and a 14KB binary stripped (no standard library included). +So Rust can actually end up with smaller binaries than C on some occasions, +but in reality I would honestly expect the same size (if not slightly bigger) +binary sizes for Rust for most cases.</p> +<h2 id="compilerfrontend">Compiler/Frontend</h2> +<p>The Rust compiler is probably one of Rust&rsquo;s biggest strengths compared to C. +The Rust compiler is a very helpful tool for developers. There are other just as +helpful tools that the creators of Rust provide separately like <a href="https://github.com/rust-lang/rustfmt">rustfmt</a> +(like gofmt if you have used Go before), <a href="https://github.com/rust-lang/rust-clippy">Clippy</a>, +and more. But the Rust compiler is probably one of the most straightforward +and user friendly (in a mostly good way) compilers that I have ever seen. +Just take a look at the difference between an error message in GCC v.s. an +error message in RustC. The same mistake is being made in both languages.</p> +<p>An error message from GCC:</p> +<pre tabindex="0"><code>test.c: In function &#39;main&#39;: +test.c:8:32: warning: passing argument 1 of &#39;sumOfSquares&#39; makes integer from pointer without a cast +[-Wint-conversion] + 8 | int32_t result = sumOfSquares(&#34;Number&#34;); + | ^~~~~~~~ + | | + | char * +test.c:3:30: note: expected &#39;int32_t&#39; {aka &#39;int&#39;} but argument is of type &#39;char *&#39; + 3 | int32_t sumOfSquares(int32_t x, int32_t y) { + | ~~~~~~~~^ +test.c:8:19: error: too few arguments to function &#39;sumOfSquares&#39; + 8 | int32_t result = sumOfSquares(&#34;Number&#34;); + | ^~~~~~~~~~~~ +test.c:3:9: note: declared here + 3 | int32_t sumOfSquares(int32_t x, int32_t y) { + | ^~~~~~~~~~~~ +</code></pre><p>An error message from RustC:</p> +<pre tabindex="0"><code>error[E0061]: this function takes 2 arguments but 1 argument was supplied + --&gt; test.rs:6:19 + | +6 | let _result = sum_of_squares(&#34;Number&#34;); + | ^^^^^^^^^^^^^^---------- + | || + | |expected `i32`, found `&amp;str` + | an argument of type `i32` is missing + | +note: function defined here + --&gt; test.rs:1:4 + | +1 | fn sum_of_squares(x: i32, y: i32) -&gt; i32 { + | ^^^^^^^^^^^^^^ ------ ------ +help: provide the argument + | +6 | let _result = sum_of_squares(/* i32 */, /* i32 */); + | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0061`. +</code></pre><p>These error messages may look similar, but in my opinion the error/warning +messages from RustC are just a bit more clear than the messages from GCC and +definitely a lot clearer than other compilers.</p> +<p>When it comes to the Rust frontend, LLVM, I don&rsquo;t really mind it as a frontend. +For the Linux kernel, I think that GCC would be a better frontend for Rust to +use and there is work being done on just <a href="https://github.com/Rust-GCC/gccrs">that</a>. +But many languages use like <a href="https://dlang.org/">D</a> (<a href="https://github.com/ldc-developers/ldc">LDC</a>), newer versions of <a href="https://www.haskell.org/">Haskell</a>, <a href="https://ziglang.org/">Zig</a>, and +many other languages use LLVM and they generate pretty good results most of +the time. Is it the most ideal frontend when working with the Linux kernel, +probably not, but is it the worst? Definitely not.</p> +<h2 id="memory-safety">Memory Safety</h2> +<p>What everyone knows Rust for, it&rsquo;s memory safety. The biggest tool that Rust +has at it&rsquo;s disposal to try to guarantee memory safety is the borrow checker. +For those of you who don&rsquo;t know what the borrow checker is, the borrow checker +assigns memory to a variable known as an owner. Once the owner of a piece of +data is out of scope, it deallocates the memory. You can also &ldquo;borrow&rdquo; the +memory (hence the name &ldquo;borrow checker), but I won&rsquo;t go too into detail, +since there are plenty of better explaniations of Rust&rsquo;s borrow checker. Just +keep in mind that the borrow checker is very important for memory safety.</p> +<p>I think that memory safety is Rust&rsquo;s biggest and most important contribution +to the Linux kernel. Memory safety can eliminate a lot if not the majority of +bugs out there in the wild and I think that the Linux kernel is no exception. +If used correctly, I think that Rust&rsquo;s memory safety could make the Linux +kernel even more robust than it already is with little to no sacrifices.</p> +<h2 id="conclusion">Conclusion</h2> +<p>Overall, I think that the inclusion of Rust in the Linux kernel isn&rsquo;t too bad +of a decision. Don&rsquo;t get me wrong, there are some bad or not fully fleshed +out parts to the language, but otherwise it&rsquo;s pretty solid. Now I&rsquo;m no expert +programmer in either C or Rust, but I think that there could be a very good +balance within the Linux kernel if Rust is used correctly.</p> +<p>That being said, I&rsquo;m personally still not sold on Rust&rsquo;s inclusion into the Linux +kernel. Despite the advantages of memory safety and a better compiler there +are still lots of parts of Rust that are not fully fleshed out and could +cause various problems for the Linux kernel. Although I think that Rust is +the best choice of any language out there to add to the Linux kernel, +I personally don&rsquo;t think that another language needs to be added to the Linux +kernel. I guess all that I can do is wait and see if this addition to the Linux +kernel is a good one or not.</p> + + + + + The Importance of Lisp + https://brycev.com/blog/the-importance-of-lisp/ + Tue, 13 Sep 2022 00:00:00 +0000 + https://brycev.com/blog/the-importance-of-lisp/ + <p>Lisp (also known as LISP) is a family of programming languages that have had a +significant impact on the world of computing. Lisp has had and still +has a great influence on the evolution of programming languages and computing +theory as a whole. It remains a very easy language to learn and not that hard +to master.</p> +<p>For those of you who don&rsquo;t know what Lisp is, Lisp stands for &ldquo;LISt Processor&rdquo;. +A Lisp program is made up of at least of at least one &ldquo;list&rdquo; which is +indicated by an expression surrounded by parentheses. So for example, +if you wanted to multiply 2 and 4, you would give an expression like this +to a Lisp interpreter or compiler:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(* <span style="color:#ae81ff">2</span> <span style="color:#ae81ff">4</span>) <span style="color:#75715e">; Results in 8</span> +</span></span></code></pre></div><p>The Lisp interpreter or compiler evaluates the contents of the list to +produce a result. The first element of the list is the operator being used, +in this case its multiplication (<code>*</code>). Any other items in the list are +arguments given to the operator, in this case <code>2</code> and <code>4</code>. If you are familiar +with Polish Notation or Prefix Notation then this may look very similar, the +only difference being the addition of parentheses to enclose the expression.</p> +<p>We can easily create compound expressions by just putting lists inside of +lists like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(- <span style="color:#ae81ff">10</span> (/ <span style="color:#ae81ff">4</span> <span style="color:#ae81ff">2</span>)) <span style="color:#75715e">; Equivalent to 10 - (4 / 2)</span> +</span></span><span style="display:flex;"><span>(+ (* <span style="color:#ae81ff">2</span> <span style="color:#ae81ff">2</span>) (* <span style="color:#ae81ff">2</span> <span style="color:#ae81ff">5</span>)) <span style="color:#75715e">; Equivalent to (2 * 2) + (2 * 5)</span> +</span></span></code></pre></div><p>It takes a bit of getting used to, but Lisp&rsquo;s notation makes it very easy to +string together compound expressions.</p> +<h2 id="practical-lisp">Practical Lisp</h2> +<p>Nowadays there are two different major dialects of Lisp, <strong>Common Lisp</strong> and +<strong>Scheme</strong>. There are not that many differences between Common Lisp and +Scheme so I will use the Scheme dialect in the rest of these examples since +I am familiar with it.</p> +<p>If we want to, we can define custom operators, functions, or variables just +like in any other programming language. To do so in Scheme we use the +<code>define</code> keyword:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(<span style="color:#66d9ef">define </span>x <span style="color:#ae81ff">24</span>) <span style="color:#75715e">; Define the variable x as 24</span> +</span></span><span style="display:flex;"><span>(<span style="color:#66d9ef">define </span>mult *) <span style="color:#75715e">; Define the operator mult as multiplication</span> +</span></span><span style="display:flex;"><span>(<span style="color:#66d9ef">define </span>(<span style="color:#a6e22e">circum</span> radius) (* <span style="color:#ae81ff">2</span> <span style="color:#ae81ff">3.14</span> radius)) <span style="color:#75715e">; Define circumference as 2 * ฯ€ * radius</span> +</span></span></code></pre></div><p>You may have noticed that we defined all three of these with the same <code>define</code> +keyword, so what makes a variable different from a function? Nothing really. +The great thing about Lisp is that almost everything defined under a <code>define</code> +statement is treated the same. This means that almost anything we can do with +variables we can do with functions and vice versa. So, for example, we can +create a function like this:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(<span style="color:#66d9ef">define </span>(<span style="color:#a6e22e">applyFunc</span> func arg1 arg2) (<span style="color:#a6e22e">func</span> arg1 arg2)) +</span></span></code></pre></div><p>This function, <code>applyFunc</code>, takes three arguments: a function and two arguments +for the given function. It then applies the two arguments to the given function +and spits out a result. We can test our <code>applyFunc</code> function like so:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(<span style="color:#66d9ef">define </span>(<span style="color:#a6e22e">sum-of-squares</span> x y) (+ (* x x) (* y y))) <span style="color:#75715e">; Same as x^2 + y^2</span> +</span></span><span style="display:flex;"><span>(<span style="color:#a6e22e">applyFunc</span> sum-of-squares <span style="color:#ae81ff">2</span> <span style="color:#ae81ff">4</span>) +</span></span></code></pre></div><p>Now this may look fairly useless at the moment because we can just call +<code>sum-of-squares</code> directly and pass it arguments manually. However, being able +to pass functions as arguments opens a whole new world of possibilities when +it comes to programming.</p> +<h2 id="higher-order-functions">Higher-Order Functions</h2> +<p>This brings us to an important topic, anonymous functions using the <code>lambda</code> +keyword. The <code>lambda</code> keyword creates a one-time use function, this can be +very useful for passing to another function as an argument. As an example we +can use a lambda function with our <code>applyFunc</code> function.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(<span style="color:#a6e22e">applyFunc</span> (<span style="color:#66d9ef">lambda </span>(<span style="color:#a6e22e">x</span> y) (* x y)) <span style="color:#ae81ff">2</span> <span style="color:#ae81ff">4</span>) <span style="color:#75715e">; Same a 2 * 4</span> +</span></span></code></pre></div><p>The example above creates a one-time use function <code>(lambda (x y) (* x y))</code> +and passes it to <code>applyFunc</code> with the arguments <code>2</code> and <code>4</code>. Although the +result isn&rsquo;t that spectacular, the implications of lambda functions opens up +a whole new world of possibilities for programming that only exist in +functional languages like Lisp. Functions that take functions or return +functions are called <code>higher-order functions</code>.</p> +<p>Lisp was one of, if not the first language to implement higher-order functions +in a programming language. Other high-level programming languages like Lua, +Python, Haskell, and more followed suit much later.</p> +<h2 id="recursion">Recursion</h2> +<p>Another important concept pioneered by Lisp in the world of programming is +recursion. A function is a recursive function when it calls itself, pretty +simple. Almost all recursive functions also have a base-case or exit clause +in order to stop recurring, otherwise they run forever. +If you still don&rsquo;t understand recursion then read this sentence again. ;)</p> +<p>As an example, let&rsquo;s translate the equation for Fibonacci numbers into a +Lisp expression. The mathematical definition is a follows:</p> + + +<figure ><img src="https://brycev.com/p/fib.webp" title="Recursive Fibonacci Equation" alt="Fibonacci Equation"></figure> + +<p>A translation from mathematical notation into Lisp would look like this:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-scheme" data-lang="scheme"><span style="display:flex;"><span>(<span style="color:#66d9ef">define </span>(<span style="color:#a6e22e">fib</span> n) <span style="color:#75715e">; Begin function</span> +</span></span><span style="display:flex;"><span>(<span style="color:#66d9ef">if </span>(&lt;= n <span style="color:#ae81ff">1</span>) n <span style="color:#75715e">; If n &lt;= 1 then return n</span> +</span></span><span style="display:flex;"><span>(+ (<span style="color:#a6e22e">fib</span> (- n <span style="color:#ae81ff">1</span>)) (<span style="color:#a6e22e">fib</span> (- n <span style="color:#ae81ff">2</span>))))) <span style="color:#75715e">; Else return fib(n-1) + fib(n-2)</span> +</span></span></code></pre></div><p>As you can see, the <code>fib</code> function defined above calls itself until it meets +the base requirements to end the recursive loop. Recursion makes it stupidly +easy to translate recursive mathematical functions to Lisp expressions. +Just like higher-order functions, Lisp was one of the first programming languages +to implement this and other languages followed afterwards.</p> +<h2 id="hold-up">Hold Up</h2> +<p>Now, before you go out and write your next project in Lisp, you should keep +something in mind. Lisp is <strong>not</strong> the fastest or smallest language out there, it +was not designed to be so. There are some pretty good implementations of Lisp +out in the wild, but don&rsquo;t expect them to outperform C, Go, Lua, or even +Python most of the time.</p> +<p>If you&rsquo;re going to program something in Lisp you should keep these things in +mind:</p> +<ul> +<li>How important is execution speed?</li> +<li>How important is program/runtime size?</li> +<li>How important is portability?</li> +</ul> +<p>If you value any three of these too much, then you might not want to write +your program in Lisp. You probably don&rsquo;t want to write a program that only +takes up a few kilobytes in Lisp, nor would you want to write a program that +needs millisecond speed either. However for all other needs Lisp works +perfectly fine if not exceptionally great!</p> +<p>But this brings us back to the importance of Lisp altogether. Most people +look at Lisp and see a language that innovated countless things in the field +of programming languages, but eventually got replaced by newer languages. But +Lisp as a language is still very much alive and making improvements and +innovations to this day. It is still a very good programming language for +tackling many tasks and remains one of my favorite programming languages.</p> + + +<figure ><a href="https://brycev.com/p/lisp.webp"><img src="https://brycev.com/p/lisp.webp" alt="LISP machine"></a></figure> + + + + + + Switching to Hugo + https://brycev.com/blog/switching-to-hugo/ + Sat, 10 Sep 2022 00:00:00 +0000 + https://brycev.com/blog/switching-to-hugo/ + <p>Over the last couple of days I have been migrating my blog from using a +markdown/Pandoc blog system to using <a href="https://gohugo.io/">Hugo</a> for my entire website.</p> +<p>For those of you who don&rsquo;t know what Hugo is, Hugo is a static site generator +that is written in <a href="https://go.dev/">Go</a>. It takes plain markdown and converts +it to plain HTML and CSS just like Pandoc. The biggest difference it has from +Pandoc is that Hugo is written specifically for creating static sites. To put +it bluntly, Hugo does one thing, and it does it well.</p> +<p>I might make a video on Hugo on some point as there&rsquo;s actually a lot of depth +to it and it&rsquo;s very customizable and extensible. But despite having a lot of +depth to it, Hugo generates relatively small and fast static webpages. +It&rsquo;s honestly a very good static site generator and I will be using it over my +markdown/Pandoc blog system from now on.</p> + + + + + Changing My Website Host + https://brycev.com/blog/changing-my-website-host/ + Sun, 26 Jun 2022 00:00:00 +0000 + https://brycev.com/blog/changing-my-website-host/ + <p>For the longest time I have used <a href="https://www.infinityfree.net/">Infinity Free</a> as the host for my website since it has free website hosting. +However, I havenโ€™t been able to successfully set up a SSL certificate without using Cloudflare. +Cloudflare has not had a good reputation and, in my opinion, <a href="https://www.unixsheikh.com/articles/stay-away-from-cloudflare.html">cannot be trusted</a> <a href="https://www.devever.net/~hl/cloudflare">at all</a>.</p> +<h2 id="1-sourcehut">1. <a href="https://sourcehut.org/">Sourcehut</a></h2> +<p>Sourcehut provides static website hosting, git repository hosting, gemini hosting (sadly no gopher hosting) and many other things. +The upside of using Sourcehut versus a lot of other platforms for hosting my website is that they use only free and open source software for hosting.</p> +<p>Hosted websites are automatically outfitted with SSL certificates which reduces the hassle (and they donโ€™t use Cloudflare). +My git repositories are also hosted at Sourcehut so it would make sense to move my website there. +Sourcehut also provides paste bins, mailing lists, wikis, and more useful tools.</p> +<p>One caveat is that, even though service is currently 100% free (as in free beer), once Sourcehut is out of alpha, there will probably be a price tag associated with all the services (although, right now the price doesn&rsquo;t look that bad.)</p> +<h2 id="2-get-a-vps">2. Get a VPS</h2> +<p>This may be my best option in the long run as itโ€™s the closest thing to self hosting that I can get. +I can host my website, git repositories, as well as almost anything else that I like including chat servers, forums, email, and etc.</p> +<p>The biggest downside of this is the price, which can vary from a couple dozen dollars a year to a few <strong>hundred</strong> dollars a year.</p> +<h2 id="3-move-to-some-other-3rd-party-static-site-host">3. Move to some other 3rd party static site host</h2> +<p>Iโ€™m probably more likely to self host my website before I resort to hosting it using <em>another</em> not well known 3rd party service. +My trust in most 3rd party hosts is slowly dwindling as the years go by.</p> +<h2 id="the-verdict">The verdict</h2> +<p>I currently plan to move my website to Sourcehut hosting for the time being, however I plan to eventually self host everything I need and become self reliant. +I also might move my website to a more elegant static site generator as writing and managing everything from scratch is starting to make everything a bit more cumbersome.</p> + + + + + Corebooting a Thinkpad X220 + https://brycev.com/blog/corebooting-a-thinkpad-x220/ + Tue, 31 May 2022 00:00:00 +0000 + https://brycev.com/blog/corebooting-a-thinkpad-x220/ + + +<figure ><img src="https://brycev.com/p/thinkpad.webp" alt="My Thinkpad X220"><figcaption>My Thinkpad X220</figcaption></figure> + +<h2 id="you-need">You Need</h2> +<ul> +<li>A Thinkpad X220</li> +<li>A Raspberry Pi</li> +<li>Female to female jumper wires</li> +<li>SOIC8 test clip</li> +<li>Another computer</li> +</ul> +<h2 id="disassembly">Disassembly</h2> +<p>For disassembly you can watch my video <a href="https://www.youtube.com/watch?v=hERguULT7Vo">here</a>.</p> +<p>But youโ€™ll just have to remove all the screws with the keyboard icon and all the screws with the box(ish) icon. +(Like I said, you can watch the video).</p> +<h2 id="attaching-the-clip-to-the-bios-chip">Attaching the clip to the BIOS chip</h2> +<p>In order to actually read/write to the BIOS chip you need to attach the SOIC8 clip to the bios chip.</p> +<h3 id="x220-bios-pinout">X220 BIOS pinout</h3> +<pre tabindex="0"><code> ______ + MOSI 5 --| |-- 4 GND + CLK 6 --| BIOS |-- 3 No Connection +No Connection 7 --| |-- 2 MISO + VCC (3.3V) 8 --|______|-- 1 CS +</code></pre><h3 id="raspberry-pi-pinout">Raspberry Pi pinout</h3> +<pre tabindex="0"><code> CS GND + 2 | | 40 ++-----------------------v-----v-----------+ +| x x x x x x x x x x x x x x x x x x x x | +| x x x x x x x x x x x x x x x x x x x x | ++-----------------^-^-^-^-----------------+ + 1 | | | | 39 + VCC | | CLK + MOSI/ \MISO +</code></pre><h2 id="setting-up-the-raspberry-pi">Setting up the Raspberry Pi</h2> +<p>Make sure to update your Raspberry PI and install and the needed packages as well as flashrom using these commands:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>sudo apt-get update <span style="color:#f92672">&amp;&amp;</span> sudo apt-get upgrade +</span></span><span style="display:flex;"><span>sudo apt-get install build-essential pciutils usbutils libpci-dev libusb-dev libftdi1 libftdi-dev zlib1g-dev +</span></span><span style="display:flex;"><span>git clone https://review.coreboot.org/flashrom.git +</span></span><span style="display:flex;"><span>cd flashrom +</span></span><span style="display:flex;"><span>make -j3 <span style="color:#f92672">&amp;&amp;</span> sudo make install +</span></span></code></pre></div><p>Now we need to download the Coreboot repo on our Raspberry PI.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>git clone --recursive https://review.coreboot.org/coreboot.git ~/coreboot +</span></span></code></pre></div><p>Next we need to install ifdtool on the Raspberry PI, you can do that by running:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>cd ~/coreboot/util/ifdtool +</span></span><span style="display:flex;"><span>make -j3 <span style="color:#f92672">&amp;&amp;</span> sudo make install +</span></span></code></pre></div><h2 id="reading-the-bios">Reading the BIOS</h2> +<p>First, we are going to create an alias so we donโ€™t need to type in a long drawn out command every time we want to read/write to the BIOS.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>alias fr<span style="color:#f92672">=</span><span style="color:#e6db74">&#39;sudo flashrom -p linux_spi:dev=/dev/spidev0.0,spispeed=1024&#39;</span> +</span></span></code></pre></div><p>Now we can get the name of our BIOS chip by just running <code>fr</code>.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>fr +</span></span></code></pre></div><p>The output should give you multiple chip names. +All of these are the same chip just with different names so you can use any of them, mine is โ€œMX25L6405โ€. +We are going to use this to set a <code>CHIP</code> variable.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>CHIP<span style="color:#f92672">=</span><span style="color:#e6db74">&#34;MX25L6405&#34;</span> +</span></span></code></pre></div><p>We are now ready to read the flash from the BIOS chip. +We are going to do this a few times in order to make sure that the connection is consistent when reading and writing.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -r flash01.bin +</span></span><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -r flash02.bin +</span></span><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -r flash03.bin +</span></span><span style="display:flex;"><span>md5sum flash01.bin flash02.bin flash03.bin +</span></span></code></pre></div><p>The output for <code>md5sum</code> for all three of the files should be exactly the same. +If the checksum for all three files are not the same then <strong>DO NOT CONTINUE!!!</strong> +Make sure that your connection is good and retry until everything reads correctly. +(If necessary, the spispeed can be lowered from 1024 for a more reliable read).</p> +<h2 id="optional-removing-the-management-engine">(Optional) Removing the management engine</h2> +<p>First we need to download me_cleaner.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>git clone https://github.com/corna/me_cleaner ~/me_cleaner +</span></span></code></pre></div><p>Now we can run me_cleaner on our flash file, in this case I will be using <code>flash01.bin</code>.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>~/me_cleaner/me_cleaner.py -S flash01.bin +</span></span></code></pre></div><p>If all goes well you should see a message that says: <code>Done! Good Luck!</code></p> +<h2 id="separating-the-image">Separating the image</h2> +<p>Now we can run ifdtool on our flash image in order to separate it.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>ifdtool -x flash01.bin +</span></span></code></pre></div><p>You should now have four different <code>.bin</code> files: 1. <code>flashregion_0_flashdescriptor.bin</code> 2. <code>flashregion_1_bios.bin</code> (Not needed) 3. <code>flashregion_2_intel_me.bin</code> 4. <code>flashregion_3_gbe.bin</code></p> +<p>We can now rename all the files to have a shorter name.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>mv flashregion_0_descriptor.bin descriptor.bin +</span></span><span style="display:flex;"><span>mv flashregion_2_intel_me.bin me.bin +</span></span><span style="display:flex;"><span>mv flashregion_3_gbe.bin gbe.bin +</span></span></code></pre></div><h2 id="setting-up-coreboot">Setting up Coreboot</h2> +<p>If you want to compile Coreboot on your Raspberry PI you can go ahead, however it might take anywhere from a few hours to a few <strong>DAYS</strong>, so be warned. +I copied my โ€œ.binโ€ files to my laptop in order to compile faster.</p> +<p>Now we want to download the Coreboot repo onto our computer that we are compiling Coreboot on. +(This may take a while). +(If you are compiling Coreboot on your Raspberry PI you can skip this).</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>git clone --recursive https://review.coreboot.org/coreboot.git ~/coreboot +</span></span></code></pre></div><blockquote> +<h2 id="optional-downloading-vga-bios">(Optional) Downloading VGA BIOS</h2> +<p>Windows and some Linux distributions rely on the VGA BIOS in order to display video. +So you can optionally download it if you need it.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>curl -fLO <span style="color:#e6db74">&#34;https://github.com/thetarkus/x220-coreboot-guide/raw/master/vga-8086-0126.bin&#34;</span> +</span></span></code></pre></div></blockquote> +<p>Now we need to make a directory to place our โ€œ.binโ€ files.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>mkdir -p ~/coreboot/3rdparty/blobs/mainboard/lenovo/x220 +</span></span><span style="display:flex;"><span>mv descriptor.bin ~/coreboot/3rdparty/blobs/mainboard/lenovo/x220/ +</span></span><span style="display:flex;"><span>mv me.bin ~/coreboot/3rdparty/blobs/mainboard/lenovo/x220/ +</span></span><span style="display:flex;"><span>mv gbe.bin ~/coreboot/3rdparty/blobs/mainboard/lenovo/x220/ +</span></span></code></pre></div><h2 id="configuring-coreboot">Configuring Coreboot</h2> +<p>On the computer youโ€™re compiling Coreboot with, youโ€™ll need to install these development packages (or their equivalents). +On Ubuntu, Debian, or any derivative you can run:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>sudo apt-get install git build-essential gnat flex bison libncurses5-dev wget zlib1g-dev +</span></span></code></pre></div><p>On Void Linux (what I use) I ran:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>sudo xbps-install git base-devel ncurses-devel wget zlib-devel gcc-ada +</span></span></code></pre></div><p>Now we can go into the Coreboot directory and run make <code>nconfig</code>.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>cd ~/coreboot +</span></span><span style="display:flex;"><span>make nconfig +</span></span></code></pre></div><p>You should see a menu pop up, now we can configure our Coreboot build. +Below is a list of what needs to be enabled, you can leave the rest of the settings just the way they are.</p> +<pre tabindex="0"><code>General Setup + - [*] Compress ramstage with LZMA + - [*] Include coreboot .config file into the ROM image + - [*] Allow use of binary-only repository + +Mainboard + - Mainboard vendor (Lenovo) + - Mainboard model (Thinkpad X220) + - ROM chip size (8192 KB (8 MB)) + - (0x100000) Size of CBFS filesystem in ROM + +Chipset + - [*] Enable VMX for virtualization + - Include CPU microcode in CBFS (Generate from tree) + - Flash ROM locking on S3 resume (Don&#39;t lock ROM sections on S3 resume) + - [*] Add Intel descriptor.bin file + (3rdparty/blobs/mainboard/$(MAINBOARDDIR)/descriptor.bin) Path and filename of the descriptor.bin file + - [*] Add Intel ME/TXE firmware + (3rdparty/blobs/mainboard/$(MAINBOARDDIR)/me.bin) Path to management engine firmware + - [*] Add gigabit ethernet firmware + (3rdparty/blobs/mainboard/$(MAINBOARDDIR)/gbe.bin) Path to gigabit ethernet firmware + +Devices + - Graphics initialization (Run VGA Option ROMs) + - [*] Use native graphics initialization + - [*] Add a VGA BIOS image + (/home/$USER/vga-8086-0126.bin) VGA BIOS path and filename + (8086,0126) VGA device PCI IDs + +Generic Drivers + - [*] PS/2 keyboard init + - [*] Support Intel PCI-e WiFi adapters + +Console + - [*] Squelch AP CPUs from early console. + - [*] Show POST codes on the debug console + +System tables + - [*] Generate SMBIOS tables + +Payload + - Add a payload (SeaBIOS) + - SeaBIOS version (master) + - (3000) PS/2 keyboard controller initialization timeout (milliseconds) + - [*] Harware init during option ROM execution + - [*] Include generated option rom that implements legacy VGA BIOS compatibility + - [*] Use LZMA compression for payloads +</code></pre><p>You can press <code>F6</code> to save your config and then press <code>F9</code> to exit. +Now we can actually compile Coreboot now.</p> +<blockquote> +<h3 id="optional-create-cross-compiler">(Optional) Create Cross Compiler</h3> +<p>If you donโ€™t have an <code>i386</code> cross compiler you can make one by running:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>make crossgcc-i386 +</span></span><span style="display:flex;"><span>make iasl +</span></span></code></pre></div></blockquote> +<p>Letโ€™s compile coreboot by running:</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>make -j<span style="color:#66d9ef">$(</span>nproc<span style="color:#66d9ef">)</span> +</span></span></code></pre></div><p>This might take a while.</p> +<p><strong>NOTE:</strong> If you canโ€™t compile Coreboot, try checking and making sure you did everything correctly.</p> +<h2 id="flashing-coreboot">Flashing Coreboot</h2> +<p><strong>WARNING: Proceed with caution, you can possibly brick your computer if you are not careful!!!</strong></p> +<p>You should now be left with a file named <code>coreboot.rom</code> in the <code>~/coreboot</code> directory. +You can copy this file back to your Raspberry PI into order to flash it.</p> +<p>Now letโ€™s go ahead and read our flash chip again to make sure that our connection is still good.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -r flash01.bin +</span></span><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -r flash02.bin +</span></span><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -r flash03.bin +</span></span><span style="display:flex;"><span>md5sum flash01.bin flash02.bin flash03.bin +</span></span></code></pre></div><p>And, like before, if all the checksums match, you can go ahead and flash <code>coreboot.rom</code>.</p> +<div class="highlight"><pre tabindex="0" style="color:#f8f8f2;background-color:#272822;-moz-tab-size:4;-o-tab-size:4;tab-size:4;"><code class="language-sh" data-lang="sh"><span style="display:flex;"><span>fr -c <span style="color:#e6db74">&#34;</span>$CHIP<span style="color:#e6db74">&#34;</span> -w coreboot.rom +</span></span></code></pre></div><p>Now, for the moment of truth, go ahead and boot your Thinkpad. +If it wonโ€™t boot, donโ€™t sweat, just rebuild and try again. +If Coreboot wonโ€™t work on your Thinkpad no matter what you try, then you can just flash a backup of the BIOS that you read earlier and your computer should work just fine.</p> +<h2 id="aftermath">Aftermath</h2> +<p><strong>Congrats!!!</strong> You successfully freed your computer from the spying eyes of Intel and your local three letter government agency. +You can now enjoy your computing in peace.</p> +<h3 id="contact">Contact</h3> +<p>If you have any questions or comments you can find my contact info <a href="https://brycevandegrift.xyz/contact/">here</a>.</p> + + + + + Website Redesign + https://brycev.com/blog/website-redesign/ + Mon, 07 Mar 2022 00:00:00 +0000 + https://brycev.com/blog/website-redesign/ + <p>I have decided to redesign most of my website. +Itโ€™s mostly a facelift, but there are a couple under-the-hood changes done.</p> +<p>I have also decided to officially add a link to my <a href="https://www.youtube.com/channel/UCOSqzSTg4QZXdi7jvV-9rUg">YouTube channel</a> on my homepage, as well as add a link to my <a href="https://sr.ht/~bpv/">Sourcehut page</a> and my <a href="https://brycevandegrift.xyz/rss.xml">rss feed</a>.</p> +<p>I plan to reorganize my website as time goes on, but for now a simple redesign will do. +I also plan to at least convert my homepage to <a href="https://www.w3schools.com/Html/html_xhtml.asp">xhtml</a> in the future, as I think that I would provide a bit more compatibility with web browsers.</p> +<p>As time goes on I will make changes to my website as needed.</p> +<h2 id="a-note-about-github-and-gitlab">A note about GitHub and GitLab</h2> +<p>If you donโ€™t know already, I am not a big fan of GitHub. +I donโ€™t like having a GitHub account, but itโ€™s almost necessary for collaborating on projects. +I have updated my <a href="https://github.com/BryceVandegrift">GitHub account</a> for hosting my git repos. +Donโ€™t expect me to answer issues or accept code from these platforms (for now (although that might change)), for that please visit my Sourcehut page.</p> + + + + + New Youtube Channel + https://brycev.com/blog/new-youtube-channel/ + Sun, 20 Feb 2022 00:00:00 +0000 + https://brycev.com/blog/new-youtube-channel/ + <p>Recently, I have decided to start up a YouTube channel. +I mostly plan to upload informative videos, but things could always change. +Right now, I am uploading videos about GNU/Linux and other technical topics related to technology, however I believe that I will expand in the future.</p> +<p>In the future, I plan to upload all of my past and future videos onto Odysee/LBRY as I think itโ€™s a superior platform compared to YouTube. +I might even consider uploading to other niche video sites eventually.</p> +<p>You can view my first video about GnuPG (GPG) <a href="https://youtu.be/GhiLR4zRqMI">here</a>.</p> +<p><a href="https://www.youtube.com/channel/UCOSqzSTg4QZXdi7jvV-9rUg">YouTube link</a></p> + + + + + Personal Update + https://brycev.com/blog/personal-update/ + Tue, 01 Feb 2022 00:00:00 +0000 + https://brycev.com/blog/personal-update/ + <p>Recently, I have been pretty busy with college classes and havenโ€™t had much time to work on personal projects. +In addition to classes I have also been busy with other personal aspects of life which I would not like to disclose currently. +I will be active again once I finish the semester (hopefully).</p> +<p>If anything comes up, feel free to email me here: <a href="https://brycevandegrift.xyz/contact/">https://brycevandegrift.xyz/contact/</a></p> + + + + + My Thoughts on BSDs + https://brycev.com/blog/my-thoughts-on-bsds/ + Fri, 31 Dec 2021 00:00:00 +0000 + https://brycev.com/blog/my-thoughts-on-bsds/ + <p>For those of you who are not a big fan of Linux but want something that caters more to the Unix philosophy, a distribution of BSD (Berkeley Software Distribution) may be right for you. +Unlike Linux, most BSD systemsโ€™ tools, kernels, and programs are built from the ground up for that specific BSD distribution. +This also means that it can take a while for a feature or program thatโ€™s popular on Linux to make its way to a specific BSD distribution. +But, BSD it is worth a try nonetheless.</p> +<p>There are four major versions of BSD that I believe to be useable as an everyday operating system: FreeBSD, OpenBSD, NetBSD, and DragonFlyBSD.</p> +<h2 id="freebsd">FreeBSD</h2> +<p><a href="https://www.freebsd.org/">FreeBSD</a> is probably the most popular BSD distribution of all time. +FreeBSD has the most packages, the most support, and the most users compared to any other BSD distribution. +FreeBSDโ€™s default file system choices are UFS (Unix File System) and ZFS (Z File System) which isnโ€™t that bad considering ZFS is a pretty reliable file system. +FreeBSD also has support of other file systems like FAT and EXT as well as Linux binary compatibility which is pretty nice. +To be honest FreeBSD has a lot of cool and interesting features that are worth checking out.</p> +<h2 id="openbsd">OpenBSD</h2> +<p><a href="https://www.openbsd.org/">OpenBSD</a> is another flavor of BSD that is focused more on security than anything other operating system. +The people behind OpenBSD are also some of the same people behind projects like OpenSSH, LibreSSL, and even tmux (to some degree). +Although OpenBSD lacks a decent amount of software and certain features, it makes up for it in system security and stability. +Overall, I would say that it is a pretty solid operating system.</p> +<h2 id="netbsd">NetBSD</h2> +<p><a href="https://netbsd.org/">NetBSD</a> is a distribution of BSD that is focused on portability more than any other operating system out there. +NetBSD is also well known for its low resource usage as well. +NetBSD not only has support for amd64, i386, and arm, but it also supports more obscure architectures like mips, powerpc, riscv, and more. +NetBSD is also very small and can be stripped down even from itโ€™s very small state. +I remember hearing rumors about NetBSD being able to run on a literal toaster in the past, which I find pretty impressive. +If you prefer size and portability over anything else, I would recommend NetBSD.</p> +<h2 id="dragonflybsd">DragonFlyBSD</h2> +<p><a href="https://www.dragonflybsd.org/">DragonFlyBSD</a> is a distribution of BSD that is tailored for performance and speed. +I believe that DragonFlyBSD definitely delivers on itโ€™s promise of speed. +Overall multi-threading and multi-processor support as well as speed is very competitive with even the fastest of Linux systems. +Swapcache, a different type of swap made for DragonFlyBSD, also helps with greatly boosting performance for large workloads. +The HAMMER file systems are also very neat. +The HAMMER file systems have instant crash recovery, snapshots, support for multiple volumes, and much more. +If you want a fast and efficient operating system, I would definitely recommend trying DragonFlyBSD.</p> + + +<figure ><img src="https://brycev.com/p/bsd.webp" title="bsd" alt="BSD"></figure> + + + + + + New Blog + https://brycev.com/blog/new-blog/ + Mon, 29 Nov 2021 00:00:00 +0000 + https://brycev.com/blog/new-blog/ + <p>Recently, I have been working on a new blog system that takes Markdown and converts it into a very simple HTML page as well as an RSS feed. +The blog system is written entirely in POSIX shell and is around 100 lines of code.</p> +<p>If successful, it should generate a blog page, an RSS feed, as well as a rolling blog index page. +The only downside is that it requires Pandoc which isn&rsquo;t that small of a package, but it makes making blog posts a lot easier to write as I can write in Markdown instead of HTML.</p> +<p>I based some of it from Luke Smithโ€™s blog script <a href="https://github.com/LukeSmithxyz/lb">here</a>. +I have added a few quirks of my own and I also plan to greatly expand on it in the future.</p> + + + + + diff --git a/sample/rss2.xml b/sample/rss2.xml new file mode 100644 index 0000000..4e4dd71 --- /dev/null +++ b/sample/rss2.xml @@ -0,0 +1,2754 @@ + + + + + ploeh blog + https://blog.ploeh.dk + danish software design + en-us + Mark Seemann + Fri, 20 Jun 2025 07:45:36 UTC + Fri, 20 Jun 2025 07:45:36 UTC + + Song recommendations from C# combinators + https://blog.ploeh.dk/2025/06/16/song-recommendations-from-c-combinators/ + Mon, 16 Jun 2025 07:41:00 UTC + + + +<div id="post"> + <p> + <em>LINQ-style composition, including SelectMany and Traverse.</em> + </p> + <p> + This article is part of a larger series titled <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">Alternative ways to design with functional programming</a>. In the <a href="/2025/06/09/song-recommendations-from-combinators">previous article</a>, I described, in general terms, a pragmatic small-scale architecture that may <em>look</em> functional, although it really isn't. + </p> + <p> + Please consult the previous articles for context about the example code base. The code shown in this article is from the <em>combinators</em> Git branch. + </p> + <p> + The goal is to extract <a href="https://en.wikipedia.org/wiki/Pure_function">pure functions</a> from the overall recommendations algorithm and compose them using standard combinators, such as <code>SelectMany</code> (<a href="/2022/03/28/monads">monadic <em>bind</em></a>), <code>Select</code>, and <code>Traverse</code>. + </p> + <h3 id="a667e64d78a346eba94571361ec22fc4"> + Composition from combinators <a href="#a667e64d78a346eba94571361ec22fc4">#</a> + </h3> + <p> + Let's start with the completed composition, and subsequently look at the most interesting parts. + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&gt;&nbsp;<span style="font-weight:bold;color:#74531f;">GetRecommendationsAsync</span>(<span style="color:blue;">string</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;_songService.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>(<span style="font-weight:bold;color:#1f377f;">userName</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">SelectMany</span>(<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;=&gt;&nbsp;<span style="color:#74531f;">UserTopScrobbles</span>(<span style="font-weight:bold;color:#1f377f;">scrobbles</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Traverse</span>(<span style="font-weight:bold;color:#1f377f;">scrobble</span>&nbsp;=&gt;&nbsp;_songService +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>(<span style="font-weight:bold;color:#1f377f;">scrobble</span>.Song.Id) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="color:#74531f;">TopListeners</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">SelectMany</span>(<span style="font-weight:bold;color:#1f377f;">users</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Traverse</span>(<span style="font-weight:bold;color:#1f377f;">user</span>&nbsp;=&gt;&nbsp;_songService +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>(<span style="font-weight:bold;color:#1f377f;">user</span>.UserName) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="color:#74531f;">TopScrobbles</span>)) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="color:#74531f;">Songs</span>))) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="color:#74531f;">TakeTopRecommendations</span>)); +}</pre> + </p> + <p> + This is a single expression with nested subexpressions. + </p> + <p> + The functions <code>UserTopScrobbles</code>, <code>TopListeners</code>, <code>TopScrobbles</code>, <code>Songs</code>, and <code>TakeTopRecommendations</code> are <code>private</code> helper functions. Here's one of them: + </p> + <p> + <pre><span style="color:blue;">private</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">IEnumerable</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&nbsp;<span style="color:#74531f;">UserTopScrobbles</span>(<span style="color:#2b91af;">IEnumerable</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>.<span style="font-weight:bold;color:#74531f;">OrderByDescending</span>(<span style="font-weight:bold;color:#1f377f;">scrobble</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>.ScrobbleCount).<span style="font-weight:bold;color:#74531f;">Take</span>(100); +}</pre> + </p> + <p> + The other helpers are also simple, single-expression functions like this one. + </p> + <p> + As <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">Oleksii Holub implies</a>, you could make each of these small functions <code>public</code> if you wished to test them individually. + </p> + <p> + Let's now look at the various building blocks that enable this composition. + </p> + <h3 id="2d6d18db84d84ada8084fd9a035918d6"> + Asynchronous monad <a href="#2d6d18db84d84ada8084fd9a035918d6">#</a> + </h3> + <p> + C# (or .NET) in general only comes with standard combinators for <a href="https://learn.microsoft.com/dotnet/api/system.collections.generic.ienumerable-1">IEnumerable&lt;T&gt;</a>, so whenever you need them for other <a href="/2022/03/28/monads">monads</a>, you have to define them yourself (or pull in a reusable library that defines them). For the above composition, you'll need <code>SelectMany</code> and <code>Select</code> for <code>Task</code> computations. You can see implementations in the article <a href="/2022/06/06/asynchronous-monads">Asynchronous monads</a>, so I'll not repeat the code here. + </p> + <p> + One exception is this extension method, which is a variant monadic <em>return</em>, which I'm not sure if I've published before: + </p> + <p> + <pre><span style="color:blue;">internal</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">T</span>&gt;&nbsp;<span style="color:#74531f;">AsTask</span>&lt;<span style="color:#2b91af;">T</span>&gt;(<span style="color:blue;">this</span>&nbsp;<span style="color:#2b91af;">T</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">source</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">FromResult</span>(<span style="font-weight:bold;color:#1f377f;">source</span>); +}</pre> + </p> + <p> + Nothing much is going on here, since it's just a wrapper of <a href="https://learn.microsoft.com/dotnet/api/system.threading.tasks.task.fromresult">Task.FromResult</a>. The <code>this</code> keyword, however, makes <code>AsTask</code> an extension method, which makes usage marginally prettier. It's not used in the above composition, but, as you'll see below, in the implementation of <code>Traverse</code>. + </p> + <h3 id="f71260c45e2b45df8a778db7182f5519"> + Traversal <a href="#f71260c45e2b45df8a778db7182f5519">#</a> + </h3> + <p> + The <a href="/2024/11/11/traversals">traversal</a> could be implemented from a hypothetical <code>Sequence</code> action, but you can also implement it directly, which is what I chose to do here. + </p> + <p> + <pre><span style="color:blue;">internal</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IEnumerable</span>&lt;<span style="color:#2b91af;">TResult</span>&gt;&gt;&nbsp;<span style="color:#74531f;">Traverse</span>&lt;<span style="color:#2b91af;">T</span>,&nbsp;<span style="color:#2b91af;">TResult</span>&gt;( +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">this</span>&nbsp;<span style="color:#2b91af;">IEnumerable</span>&lt;<span style="color:#2b91af;">T</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">source</span>, +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Func</span>&lt;<span style="color:#2b91af;">T</span>,&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">TResult</span>&gt;&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">selector</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">source</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="font-weight:bold;color:#1f377f;">selector</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Aggregate</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Enumerable</span>.<span style="color:#74531f;">Empty</span>&lt;<span style="color:#2b91af;">TResult</span>&gt;().<span style="font-weight:bold;color:#74531f;">AsTask</span>(), +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">async</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">acc</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">x</span>)&nbsp;=&gt;&nbsp;(<span style="color:blue;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">acc</span>).<span style="font-weight:bold;color:#74531f;">Append</span>(<span style="color:blue;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">x</span>)); +}</pre> + </p> + <p> + Mapping <code>selector</code> over <code>source</code> produces a sequence of tasks. The <code>Aggregate</code> expression subsequently inverts the <a href="https://bartoszmilewski.com/2014/01/14/functors-are-containers/">containers</a> to a single task that contains a sequence of result values. + </p> + <p> + That's really all there is to it. + </p> + <h3 id="b717fff9dd2b4bdb826dc62a3d3a9050"> + Conclusion <a href="#b717fff9dd2b4bdb826dc62a3d3a9050">#</a> + </h3> + <p> + In the <a href="/2025/06/09/song-recommendations-from-combinators">previous article</a>, I made no secret of my position on this refactoring. For the example at hand, the benefit is at best marginal. The purpose of this article isn't to insist that you must write code like this. Rather, it's a demonstration of what's possible. + </p> + <p> + If you have a problem which is similar, but more complicated, refactoring to standard combinators <em>may</em> be a good idea. After all, a standard combinator like <code>SelectMany</code>, <code>Traverse</code>, etc. is well-understood and lawful. You should expect combinators to be defect-free, so using them instead of ad-hoc code constructs like nested loops with conditionals could help eliminate some trivial bugs. + </p> + <p> + Additionally, if you're working with a team comfortable with these few abstractions, code assembled from standard combinators may actually turn out to be <em>more</em> readable that code buried in ad-hoc imperative <a href="https://en.wikipedia.org/wiki/Control_flow">control flow</a>. And if not everyone on the team is on board with this style, perhaps it's <a href="/2015/08/03/idiomatic-or-idiosyncratic">an opportunity to push the envelope a bit</a>. + </p> + <p> + Of course, if you use a language where such constructs are already idiomatic, colleagues should already be used to this style of programming. + </p> + <p> + <strong>Next:</strong> Song recommendations from F# combinators. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/06/16/song-recommendations-from-c-combinators + + + + Song recommendations from combinators + https://blog.ploeh.dk/2025/06/09/song-recommendations-from-combinators/ + Mon, 09 Jun 2025 14:02:00 UTC + + + +<div id="post"> + <p> + <em>Interleaving impure actions with pure functions. Not really functional programming.</em> + </p> + <p> + This article is part of a larger article series about <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">alternative ways to design with functional programming</a>, particularly when faced with massive data loads. In the previous few articles, you saw <a href="/2018/11/19/functional-architecture-a-definition">functional architecture</a> at its apparent limit. With sufficiently large data sizes, the <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> pattern starts to buckle. That's really not an indictment of that pattern; only an observation that no design pattern applies universally. + </p> + <p> + In this and the next few articles, we'll instead look at a more pragmatic option. In this article I'll discuss the general idea, and follow up in other articles with examples in three different languages. + </p> + <p> + In this overall article series, I'm using <a href="https://tyrrrz.me/">Oleksii Holub</a>'s inspiring article <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">Pure-Impure Segregation Principle</a> as an outset for the code example. Previous articles in this article series have already covered the basics, but the gist of it is a song recommendation service that uses past play information ('scrobbles') to suggest new songs to a user. + </p> + <h3 id="ae576d7ff83c4cf8b5ce96b861d3cad0"> + Separating pure functions from impure composition <a href="#ae576d7ff83c4cf8b5ce96b861d3cad0">#</a> + </h3> + <p> + In the original article, Oleksii Holub suggests a way to separate <a href="https://en.wikipedia.org/wiki/Pure_function">pure functions</a> from impure actions: We may extract as much pure code from the overall algorithm as possible, but we're still left with pure functions and impure actions mixed together. + </p> + <p> + Here's my reproduction of that suggestion, with trivial modifications: + </p> + <p> + <pre><span style="color:green;">//&nbsp;Pure</span> +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:blue;">int</span>&gt;&nbsp;<span style="color:#74531f;">HandleOwnScrobbles</span>(<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>)&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">OrderByDescending</span>(<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.ScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Take</span>(100) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.Id) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">ToArray</span>(); + +<span style="color:green;">//&nbsp;Pure</span> +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:blue;">string</span>&gt;&nbsp;<span style="color:#74531f;">HandleOtherListeners</span>(<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">User</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span>)&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Where</span>(<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount&nbsp;&gt;=&nbsp;10_000) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">OrderByDescending</span>(<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Take</span>(20) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.UserName) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">ToArray</span>(); + +<span style="color:green;">//&nbsp;Pure</span> +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&nbsp;<span style="color:#74531f;">HandleOtherScrobbles</span>(<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>)&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Where</span>(<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.IsVerifiedArtist) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">OrderByDescending</span>(<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Take</span>(10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">ToArray</span>(); + +<span style="color:green;">//&nbsp;Pure</span> +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&nbsp;<span style="color:#74531f;">FinalizeRecommendations</span>(<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span>)&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">OrderByDescending</span>(<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Take</span>(200) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">ToArray</span>(); + +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&gt;&nbsp;<span style="font-weight:bold;color:#74531f;">GetRecommendationsAsync</span>(<span style="color:blue;">string</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;_songService.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>(<span style="font-weight:bold;color:#1f377f;">userName</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songIds</span>&nbsp;=&nbsp;<span style="color:#74531f;">HandleOwnScrobbles</span>(<span style="font-weight:bold;color:#1f377f;">scrobbles</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">List</span>&lt;<span style="color:#2b91af;">Song</span>&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">foreach</span>&nbsp;(<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songId</span>&nbsp;<span style="font-weight:bold;color:#8f08c4;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songIds</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListeners</span>&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;_songService +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>(<span style="font-weight:bold;color:#1f377f;">songId</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserNames</span>&nbsp;=&nbsp;<span style="color:#74531f;">HandleOtherListeners</span>(<span style="font-weight:bold;color:#1f377f;">otherListeners</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">foreach</span>&nbsp;(<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserName</span>&nbsp;<span style="font-weight:bold;color:#8f08c4;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserNames</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span>&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;_songService +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>(<span style="font-weight:bold;color:#1f377f;">otherUserName</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songsToRecommend</span>&nbsp;=&nbsp;<span style="color:#74531f;">HandleOtherScrobbles</span>(<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>.<span style="font-weight:bold;color:#74531f;">AddRange</span>(<span style="font-weight:bold;color:#1f377f;">songsToRecommend</span>); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;} +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="color:#74531f;">FinalizeRecommendations</span>(<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>); +}</pre> + </p> + <p> + As Oleksii Holub writes, + </p> + <blockquote> + <p> + "However, instead of having one cohesive element to reason about, we ended up with multiple fragments, each having no meaning or value of their own. While unit testing of individual parts may have become easier, the benefit is very questionable, as it provides no confidence in the correctness of the algorithm as a whole." + </p> + </blockquote> + <p> + I agree with that assessment, but still find it warranted to pursue the idea a little further. After all, my goal with this overall article series isn't to be prescriptive, but rather descriptive. By presenting and comparing alternatives, we become aware of more options. This, hopefully, helps us choose the 'right tool for the job'. + </p> + <h3 id="eba5c36720814300b4a4127359569b46"> + Triple-decker sandwich? <a href="#eba5c36720814300b4a4127359569b46">#</a> + </h3> + <p> + If we look closer at this alternative, however, we find that we only need to deal with three impure actions. We might, then, postulate that this is an <a href="/2023/10/09/whats-a-sandwich">expanded sandwich</a> - a triple-decker sandwich, if you will. + </p> + <p> + To be clear, I don't find this a reasonable argument. Even if you accept expanding the sandwich metaphor to add a pure validation step, the number of layers, and the structure of the sandwich would still be known at compile time. You may start at the impure boundary, then phase into pure validation, return to another impure step to gather data, call your 'main' pure function, and finally write the result to some kind of output. To borrow a figure from the <a href="/2023/10/09/whats-a-sandwich">What's a sandwich?</a> article: + </p> + <p> + <img src="/content/binary/pure-impure-pure-impure-box.png" alt="A box with green, red, green, and red horizontal tiers."> + </p> + <p> + On the other hand, this isn't what the above code suggestion does. The problem with the song recommendation algorithm is that the impure actions cascade. While we start with a single impure out-of-process query, we then use the result of that to loop over, and perform <em>n</em> more queries. This, in fact, happens again, nested in the outer loop, so in terms of network calls, we're looking at an <em>O(n<sup>2</sup>)</em> algorithm. + </p> + <p> + We can actually be more precise than that, because the 'outer' queries actually limit their result sets. The first query only considers the top 100 results, so we know that <code>GetTopListenersAsync</code> is going to be called at most 100 times. The result of this call is again limited to the top 20, so that the inner calls to <code>GetTopScrobblesAsync</code> run at most 20 * 100 = 2,000 times. In all, the upper limit is 1 + 100 + 2,000 = 2,101 network calls. (Okay, so really, this is just an <em>O(1)</em> algorithm, although <em>1 ~ 2,101</em>.) + </p> + <p> + Not that that isn't going to take a bit of time. + </p> + <p> + All that said, it's not execution time that concerns me in this context. Assume that the algorithm is already as optimal as possible, and that those 2,101 network calls are necessary. What rather concerns me here is how to organize the code in a way that's as maintainable as possible. As usual, when that's the main concern, I'll remind the reader to consider the example problem as a stand-in for a more complicated problem. Even Oleksii Holub's original code example is only some fifty-odd lines of code, which in itself hardly warrants all the hand-wringing we're currently subjecting it to. + </p> + <p> + Rather, what I'd like to address is the dynamic back-and-forth between pure function and impure action. Each of these thousands of out-of-process calls are non-deterministic. If you're tasked with maintaining or editing this algorithm, your brain will be taxed by all that unpredictable behaviour. Many subtle bugs lurk there. + </p> + <p> + The more we can pull the code towards pure functions the better, because <a href="/2021/07/28/referential-transparency-fits-in-your-head">referential transparency fits in your head</a>. + </p> + <p> + So, to be explicit, I don't consider this kind of composition as an expanded Impureim Sandwich. + </p> + <h3 id="4903416bdf924a29a6f6043b178da4df"> + Standard combinators <a href="#4903416bdf924a29a6f6043b178da4df">#</a> + </h3> + <p> + Is it possible to somehow improve, even just a little, on the above suggestion? Can we somehow make it look a little 'more functional'? + </p> + <p> + We could use some standard combinators, like <a href="/2022/03/28/monads">monadic <em>bind</em></a>, <a href="/2024/11/11/traversals">traversals</a>, and so on. + </p> + <p> + To be honest, for the specific song-recommendations example, the benefit is marginal at best, but doing it would still demonstrate a particular technique. We'd be able to get rid of the local mutation of <code>recommendationCandidates</code>, but that's about it. + </p> + <p> + Even so, refactoring to self-contained expressions makes other refactoring easier. As a counter-example, imagine that you'd like to extract the inner <code>foreach</code> loop in the above code example to a helper method. + </p> + <p> + <pre><span style="color:blue;">private</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">CollectOtherUserTopScrobbles</span>( +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">List</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>, +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">IReadOnlyList</span>&lt;<span style="color:blue;">string</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserNames</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">foreach</span>&nbsp;(<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserName</span>&nbsp;<span style="font-weight:bold;color:#8f08c4;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserNames</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span>&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;_songService +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>(<span style="font-weight:bold;color:#1f377f;">otherUserName</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songsToRecommend</span>&nbsp;=&nbsp;<span style="color:#74531f;">HandleOtherScrobbles</span>(<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>.<span style="font-weight:bold;color:#74531f;">AddRange</span>(<span style="font-weight:bold;color:#1f377f;">songsToRecommend</span>); +&nbsp;&nbsp;&nbsp;&nbsp;} +}</pre> + </p> + <p> + The call site would then look like this: + </p> + <p> + <pre><span style="color:green;">//&nbsp;Pure</span> +<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserNames</span>&nbsp;=&nbsp;<span style="color:#74531f;">HandleOtherListeners</span>(<span style="font-weight:bold;color:#1f377f;">otherListeners</span>); + +<span style="color:green;">//&nbsp;Impure</span> +<span style="color:blue;">await</span>&nbsp;<span style="font-weight:bold;color:#74531f;">CollectOtherUserTopScrobbles</span>(<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">otherUserNames</span>);</pre> + </p> + <p> + In this specific example, such a refactoring isn't too difficult, but it's more complicated than it could be. Because of state mutation, we have to pass the object to be modified, in this case <code>recommendationCandidates</code>, along as a method argument. Here, there's only one, but if you have code where you change the state of two objects, you'd have to pass two extra parameters, and so on. + </p> + <p> + You've most likely worked in a real code base where you have tried to extract a helper method, only to discover that it's so incredibly tangled with the objects that it modifies that you need a long parameter list. What should have been a simplification is in danger of making everything worse. + </p> + <p> + On the other hand, self-contained expressions, even if, as in this case, they're non-deterministic, don't mutate state. In general, this tends to make it easier to extract subexpressions as helper methods, if only because they are less coupled to the rest of the code. They may required inputs as parameters, but at least you don't have to pass around objects to be modified. + </p> + <p> + Thus, the reason I find it worthwhile to include articles about this kind of refactoring is that, since it demonstrates how to refactor to a more expression-based style, you may be able to extrapolate to your own context. And who knows, you may encounter a context where more substantial improvements can be made by moving in this direction. + </p> + <p> + As usual in this article series, you'll see how to apply this technique in three different languages. + </p> + <ul> + <li><a href="/2025/06/16/song-recommendations-from-c-combinators">Song recommendations from C# combinators</a></li> + <li>Song recommendations from F# combinators</li> + <li>Song recommendations from Haskell combinators</li> + </ul> + <p> + All that said, it's important to underscore that I don't consider this proper <a href="/2018/11/19/functional-architecture-a-definition">functional architecture</a>. Even the Haskell example is too non-deterministic to my tastes. + </p> + <h3 id="d470d8b3fd2344788be43691aac8a29c"> + Conclusion <a href="#d470d8b3fd2344788be43691aac8a29c">#</a> + </h3> + <p> + Perhaps the most pragmatic approach to a problem like the song-recommendations example is to allow the impure actions and pure functions to interleave. I don't mean to insist that functional programming is the only way to make code maintainable. You can organize code according to other principles, and some of them may also leave you with a code base that can serve its mission well, now and in the future. + </p> + <p> + Another factor to take into account is the skill level of the team tasked with maintaining a code base. What are they comfortable with? + </p> + <p> + Not that I think you should settle for status quo. Progress can only be made if you <a href="/2015/08/03/idiomatic-or-idiosyncratic">push the envelop a little</a>, but you can also come up with a code base so alien to your colleagues that they can't work with it at all. + </p> + <p> + I could easily imagine a team where the solution in the next three articles is the only style they'd be able to maintain. + </p> + <p> + <strong>Next:</strong> <a href="/2025/06/16/song-recommendations-from-c-combinators">Song recommendations from C# combinators</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/06/09/song-recommendations-from-combinators + + + + Testing races with a slow Decorator + https://blog.ploeh.dk/2025/06/02/testing-races-with-a-slow-decorator/ + Mon, 02 Jun 2025 08:03:00 UTC + + + +<div id="post"> + <p> + <em>Delaying database interactions for test purposes.</em> + </p> + <p> + In chapter 12 in <a href="/2021/06/14/new-book-code-that-fits-in-your-head">Code That Fits in Your Head</a>, I cover a typical <a href="https://en.wikipedia.org/wiki/Race_condition">race condition</a> and how to test for it. The book comes with a pedagogical explanation of the problem, including a diagram in the style of <a href="/ref/ddia">Designing Data-Intensive Applications</a>. In short, the problem occurs when two or more clients are competing for the last remaining seats in a particular time slot. + </p> + <p> + In my two-day workshop based on the book, I also cover this scenario. The goal is to show how to write automated tests for this kind of non-deterministic behaviour. In the book, and in the workshop, my approach is to rely on the <a href="https://en.wikipedia.org/wiki/Law_of_large_numbers">law of large numbers</a>. An automated test attempts to trigger the race condition by trying 'enough' times. A timeout on the test assumes that if the test does not trigger the condition in the allotted time window, then the bug is addressed. + </p> + <p> + At one of my workshops, one participant told me of a more efficient and elegant way to test for this. I wish I could remember exactly at which workshop it was, and who the gentleman was, but alas, it escapes me. + </p> + <h3 id="05f24d2c08e34c5fbbd7e91dc8667969"> + Reproducing the condition <a href="#05f24d2c08e34c5fbbd7e91dc8667969">#</a> + </h3> + <p> + How do you deterministically reproduce non-deterministic behaviour? The default answer is almost tautological. You can't, since it's non-deterministic. + </p> + <p> + The irony, however, is that in the workshop, I deterministically demonstrate the problem. The problem, in short, is that in order to decide whether or not to accept a reservation request, the system first reads data from its database, runs a fairly complex piece of decision logic, and finally writes the reservation to the database - if it decides to accept it, based on what it read. When competing processes vie for the last remaining seats, a race may occur where both (or all) base their decision on the same data, so they all come to the conclusion that they still have enough remaining capacity. Again, refer to the book, and its accompanying code base, for the details. + </p> + <p> + How do I demonstrate this condition in the workshop? I go into the Controller code and insert a temporary, human-scale delay after reading from the database, but before making the decision: + </p> + <p> + <pre><span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservations</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Repository.<span style="font-weight:bold;color:#74531f;">ReadReservations</span>(<span style="font-weight:bold;color:#1f377f;">r</span>.At); + +<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(<span style="color:#2b91af;">TimeSpan</span>.<span style="color:#74531f;">FromSeconds</span>(10)); + +<span style="font-weight:bold;color:#8f08c4;">if</span>&nbsp;(!MaitreD.<span style="font-weight:bold;color:#74531f;">WillAccept</span>(<span style="color:#2b91af;">DateTime</span>.Now,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservations</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">r</span>)) +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="color:#74531f;">NoTables500InternalServerError</span>(); + +<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Repository.<span style="font-weight:bold;color:#74531f;">Create</span>(<span style="font-weight:bold;color:#1f377f;">restaurant</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>);</pre> + </p> + <p> + Then I open two windows, from which I, within a couple of seconds of each other, try to make competing reservations. When the bug is present, both reservations are accepted, although, according to business rules, only one should be. + </p> + <p> + So that's how to deterministically demonstrate the problem. Just insert a long enough delay. + </p> + <p> + We can't, however, leave such delays in the production code, so I never even considered that this simple technique could be used for automated testing. + </p> + <h3 id="d908cad4489642babb34a58a37c678c7"> + Slowing things down with a Decorator <a href="#d908cad4489642babb34a58a37c678c7">#</a> + </h3> + <p> + That's until my workshop participant told me his trick: Why don't you slow down the database interactions for test-purposes only? At first, I thought he had in mind some nasty compiler pragmas or environment hacks, but no. Why don't you use a <a href="https://en.wikipedia.org/wiki/Decorator_pattern">Decorator</a> to slow things down? + </p> + <p> + Indeed, why not? + </p> + <p> + Fortunately, all database interaction already takes place behind an <code>IReservationsRepository</code> interface. Adding a test-only, delaying Decorator is straightforward. + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">sealed</span>&nbsp;<span style="color:blue;">class</span>&nbsp;<span style="color:#2b91af;">SlowReservationsRepository</span>&nbsp;:&nbsp;<span style="color:#2b91af;">IReservationsRepository</span> +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:blue;">readonly</span>&nbsp;<span style="color:#2b91af;">TimeSpan</span>&nbsp;halfDelay; + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:#2b91af;">SlowReservationsRepository</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">TimeSpan</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">delay</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">IReservationsRepository</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">inner</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Delay&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">delay</span>; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;halfDelay&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">delay</span>&nbsp;<span style="font-weight:bold;color:#74531f;">/</span>&nbsp;2; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Inner&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">inner</span>; +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:#2b91af;">TimeSpan</span>&nbsp;Delay&nbsp;{&nbsp;<span style="color:blue;">get</span>;&nbsp;} +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:#2b91af;">IReservationsRepository</span>&nbsp;Inner&nbsp;{&nbsp;<span style="color:blue;">get</span>;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">Create</span>(<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="color:#2b91af;">Reservation</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Inner.<span style="font-weight:bold;color:#74531f;">Create</span>(<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">Delete</span>(<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="color:#2b91af;">Guid</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Inner.<span style="font-weight:bold;color:#74531f;">Delete</span>(<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">Reservation</span>?&gt;&nbsp;<span style="font-weight:bold;color:#74531f;">ReadReservation</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">restaurantId</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Guid</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">result</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Inner.<span style="font-weight:bold;color:#74531f;">ReadReservation</span>(<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">result</span>; +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">Reservation</span>&gt;&gt;&nbsp;<span style="font-weight:bold;color:#74531f;">ReadReservations</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">restaurantId</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">DateTime</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">min</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">DateTime</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">max</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">result</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Inner.<span style="font-weight:bold;color:#74531f;">ReadReservations</span>(<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">min</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">max</span>); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">result</span>; +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">Update</span>(<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="color:#2b91af;">Reservation</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Inner.<span style="font-weight:bold;color:#74531f;">Update</span>(<span style="font-weight:bold;color:#1f377f;">restaurantId</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">Delay</span>(halfDelay); +&nbsp;&nbsp;&nbsp;&nbsp;} +}</pre> + </p> + <p> + This one uniformly slows down all operations. I arbitrarily decided to split the <code>Delay</code> in half, in order to apply half of it before each action, and the other half after. Honestly, I didn't mull this over too much; I just thought that if I did it that way, I wouldn't have to speculate whether it would make a difference if the delay happened before or after the action in question. + </p> + <h3 id="a0e3b21e6def42b781e47a98b19f2294"> + Slowing down tests <a href="#a0e3b21e6def42b781e47a98b19f2294">#</a> + </h3> + <p> + I added a few helper methods to the <code>RestaurantService</code> class that inherits from <a href="https://learn.microsoft.com/dotnet/api/microsoft.aspnetcore.mvc.testing.webapplicationfactory-1"><span style="color:#2b91af;">WebApplicationFactory</span>&lt;<span style="color:#2b91af;">Startup</span>&gt;</a>, mainly to enable decoration of the injected Repository. With those changes, I could now rewrite my test like this: + </p> + <p> + <pre>[<span style="color:#2b91af;">Fact</span>] +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">NoOverbookingRace</span>() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">date</span>&nbsp;=&nbsp;<span style="color:#2b91af;">DateTime</span>.Now.Date.<span style="font-weight:bold;color:#74531f;">AddDays</span>(1).<span style="font-weight:bold;color:#74531f;">AddHours</span>(18.5); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">using</span>&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>&nbsp;=&nbsp;<span style="color:#2b91af;">RestaurantService</span>.<span style="color:#74531f;">CreateWith</span>(<span style="font-weight:bold;color:#1f377f;">repo</span>&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">SlowReservationsRepository</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">TimeSpan</span>.<span style="color:#74531f;">FromMilliseconds</span>(100),&nbsp;<span style="font-weight:bold;color:#1f377f;">repo</span>)); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">task1</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">PostReservation</span>(<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">ReservationDtoBuilder</span>() +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithDate</span>(<span style="font-weight:bold;color:#1f377f;">date</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithQuantity</span>(10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Build</span>()); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">task2</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">PostReservation</span>(<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">ReservationDtoBuilder</span>() +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithDate</span>(<span style="font-weight:bold;color:#1f377f;">date</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithQuantity</span>(10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Build</span>()); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">actual</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">WhenAll</span>(<span style="font-weight:bold;color:#1f377f;">task1</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">task2</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">actual</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>.StatusCode&nbsp;==&nbsp;<span style="color:#2b91af;">HttpStatusCode</span>.InternalServerError); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">ok</span>&nbsp;=&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>(<span style="font-weight:bold;color:#1f377f;">actual</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>.IsSuccessStatusCode); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Check&nbsp;that&nbsp;the&nbsp;reservation&nbsp;was&nbsp;actually&nbsp;created:</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">GetReservation</span>(<span style="font-weight:bold;color:#1f377f;">ok</span>.Headers.Location); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">resp</span>.<span style="font-weight:bold;color:#74531f;">EnsureSuccessStatusCode</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp</span>.<span style="font-weight:bold;color:#74531f;">ParseJsonContent</span>&lt;<span style="color:#2b91af;">ReservationDto</span>&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Equal</span>(10,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>.Quantity); +}</pre> + </p> + <p> + The restaurant being tested has a maximum capacity of ten guests, so while it can accommodate either of the two requests, it can't make room for both. + </p> + <p> + For this example, I arbitrarily chose to configure the Decorator with a 100-millisecond delay. Every interaction with the database caused by that test gets a built-in 100-millisecond delay. 50 ms before each action, and 50 ms after. + </p> + <p> + The test starts both tasks, <code>task1</code> and <code>task2</code>, without awaiting them. This allows them to run concurrently. After starting both tasks, the test awaits both of them with <a href="https://learn.microsoft.com/dotnet/api/system.threading.tasks.task.whenall">Task.WhenAll</a>. + </p> + <p> + The <a href="/2013/06/24/a-heuristic-for-formatting-code-according-to-the-aaa-pattern">assertion phase</a> of the test is more involved than you may be used to see. The reason is that it deals with more than one possible failure scenario. + </p> + <p> + The first two assertions (<code>Assert.Single</code>) deal with the complete absence of transaction control in the application. In that case, both <code>POST</code> requests succeed, which they aren't supposed to. If the system works properly, it should accept one request and reject the other. + </p> + <p> + The rest of the assertions check that the successful reservation was actually created. That's another failure scenario. + </p> + <p> + The way I chose to deal with the race condition is standard in .NET. I used a <a href="https://learn.microsoft.com/dotnet/api/system.transactions.transactionscope">TransactionScope</a>. This is peculiar and, in my opinion, questionable API that enables you to start a transaction <em>anywhere</em> in your code, and then complete when you you're done. In the code base that accompanies <a href="/code-that-fits-in-your-head">Code That Fits in Your Head</a>, it looks like this: + </p> + <p> + <pre><span style="color:blue;">private</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">ActionResult</span>&gt;&nbsp;<span style="font-weight:bold;color:#74531f;">TryCreate</span>(<span style="color:#2b91af;">Restaurant</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">restaurant</span>,&nbsp;<span style="color:#2b91af;">Reservation</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">using</span>&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scope</span>&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">TransactionScope</span>(<span style="color:#2b91af;">TransactionScopeAsyncFlowOption</span>.Enabled); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservations</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Repository +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">ReadReservations</span>(<span style="font-weight:bold;color:#1f377f;">restaurant</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>.At) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">ConfigureAwait</span>(<span style="color:blue;">false</span>); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">now</span>&nbsp;=&nbsp;Clock.<span style="font-weight:bold;color:#74531f;">GetCurrentDateTime</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">if</span>&nbsp;(!<span style="font-weight:bold;color:#1f377f;">restaurant</span>.MaitreD.<span style="font-weight:bold;color:#74531f;">WillAccept</span>(<span style="font-weight:bold;color:#1f377f;">now</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservations</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>)) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="color:#74531f;">NoTables500InternalServerError</span>(); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;Repository.<span style="font-weight:bold;color:#74531f;">Create</span>(<span style="font-weight:bold;color:#1f377f;">restaurant</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>).<span style="font-weight:bold;color:#74531f;">ConfigureAwait</span>(<span style="color:blue;">false</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scope</span>.<span style="font-weight:bold;color:#74531f;">Complete</span>(); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="color:#74531f;">Reservation201Created</span>(<span style="font-weight:bold;color:#1f377f;">restaurant</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>); +}</pre> + </p> + <p> + Notice the <code>scope.Complete()</code> statement towards the end. + </p> + <p> + What happens if someone forgets to call <code>scope.Complete()</code>? + </p> + <p> + In that case, the thread that wins the race returns <code>201 Created</code>, but when the <code>scope</code> goes out of scope, it's disposed of. If <code>Complete()</code> wasn't called, the transaction is rolled back, but the HTTP response code remains <code>201</code>. Thus, the two assertions that inspect the response codes aren't enough to catch this particular kind of defect. + </p> + <p> + Instead, the test subsequently queries the System Under Test to verify that the resource was, indeed, created. + </p> + <h3 id="064b9e4646544b60b47b25e2cc4cc8a5"> + Wait time <a href="#064b9e4646544b60b47b25e2cc4cc8a5">#</a> + </h3> + <p> + The original test shown in the book times out after 30 seconds if it can't produce the race condition. Compared to that, the refactored test shown here is <em>fast</em>. Even so, we may fear that it spends too much time doing nothing. How much time might that be? + </p> + <p> + The <code>TryCreate</code> helper method shown above is the only part of a <code>POST</code> request that interacts with the Repository. As you can see, it calls it twice: Once to read, and once to write, if it decides to do that. With a 100 ms delay, that's 200 ms. + </p> + <p> + And while the test issues two <code>POST</code> requests, they run in parallel. That's the whole point. It means that they'll still run in approximately 200 ms. + </p> + <p> + The test then issues a <code>GET</code> request to verify that the resource was created. That triggers another database read, which takes another 100 ms. + </p> + <p> + That's 300 ms in all. Given that these tests are part of a second-level test suite, and not your default developer test suite, that may be good enough. + </p> + <p> + Still, that's the <code>POST</code> scenario. I also wrote a test that checks for a race condition when doing <code>PUT</code> requests, and it performs more work. + </p> + <p> + <pre>[<span style="color:#2b91af;">Fact</span>] +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">NoOverbookingPutRace</span>() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">date</span>&nbsp;=&nbsp;<span style="color:#2b91af;">DateTime</span>.Now.Date.<span style="font-weight:bold;color:#74531f;">AddDays</span>(1).<span style="font-weight:bold;color:#74531f;">AddHours</span>(18.5); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">using</span>&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>&nbsp;=&nbsp;<span style="color:#2b91af;">RestaurantService</span>.<span style="color:#74531f;">CreateWith</span>(<span style="font-weight:bold;color:#1f377f;">repo</span>&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">SlowReservationsRepository</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">TimeSpan</span>.<span style="color:#74531f;">FromMilliseconds</span>(100),&nbsp;<span style="font-weight:bold;color:#1f377f;">repo</span>)); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">address1</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">dto1</span>)&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">PostReservation</span>(<span style="font-weight:bold;color:#1f377f;">date</span>,&nbsp;4); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">address2</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">dto2</span>)&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">PostReservation</span>(<span style="font-weight:bold;color:#1f377f;">date</span>,&nbsp;4); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">dto1</span>.Quantity&nbsp;+=&nbsp;2; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">dto2</span>.Quantity&nbsp;+=&nbsp;2; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">task1</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">PutReservation</span>(<span style="font-weight:bold;color:#1f377f;">address1</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">dto1</span>); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">task2</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">PutReservation</span>(<span style="font-weight:bold;color:#1f377f;">address2</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">dto2</span>); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">actual</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">WhenAll</span>(<span style="font-weight:bold;color:#1f377f;">task1</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">task2</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">actual</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>.StatusCode&nbsp;==&nbsp;<span style="color:#2b91af;">HttpStatusCode</span>.InternalServerError); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">ok</span>&nbsp;=&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>(<span style="font-weight:bold;color:#1f377f;">actual</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>.IsSuccessStatusCode); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Check&nbsp;that&nbsp;the&nbsp;reservations&nbsp;now&nbsp;have&nbsp;consistent&nbsp;values:</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">client</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">CreateClient</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp1</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">client</span>.<span style="font-weight:bold;color:#74531f;">GetAsync</span>(<span style="font-weight:bold;color:#1f377f;">address1</span>); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp2</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">client</span>.<span style="font-weight:bold;color:#74531f;">GetAsync</span>(<span style="font-weight:bold;color:#1f377f;">address2</span>); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">resp1</span>.<span style="font-weight:bold;color:#74531f;">EnsureSuccessStatusCode</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">resp2</span>.<span style="font-weight:bold;color:#74531f;">EnsureSuccessStatusCode</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">body1</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp1</span>.<span style="font-weight:bold;color:#74531f;">ParseJsonContent</span>&lt;<span style="color:#2b91af;">ReservationDto</span>&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">body2</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp2</span>.<span style="font-weight:bold;color:#74531f;">ParseJsonContent</span>&lt;<span style="color:#2b91af;">ReservationDto</span>&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>(<span style="color:blue;">new</span>[]&nbsp;{&nbsp;<span style="font-weight:bold;color:#1f377f;">body1</span>.Quantity,&nbsp;<span style="font-weight:bold;color:#1f377f;">body2</span>.Quantity&nbsp;},&nbsp;6); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>(<span style="color:blue;">new</span>[]&nbsp;{&nbsp;<span style="font-weight:bold;color:#1f377f;">body1</span>.Quantity,&nbsp;<span style="font-weight:bold;color:#1f377f;">body2</span>.Quantity&nbsp;},&nbsp;4); +}</pre> + </p> + <p> + This test first has to create two reservations in a nice, sequential manner. Then it attempts to perform two concurrent updates, and finally it tests that all is as it should be: That both reservations still exist, but only one had its <code>Quantity</code> increased to <code>6</code>. + </p> + <p> + This test first makes two <code>POST</code> requests, nicely serialized so as to avoid a race condition. That's 400 ms. + </p> + <p> + Each <code>PUT</code> request triggers three Repository actions, for a total of 300 ms (since they run in parallel). + </p> + <p> + Finally, the test issues two <code>GET</code> requests for verification, for another 2 times 100 ms. Now that I'm writing this, I realize that I could also have parallelized these two calls, but as you read on, you'll see why that's not necessary. + </p> + <p> + In all, this test waits for 900 ms. That's almost a second. + </p> + <p> + Can we improve on that? + </p> + <h3 id="b5c26b0e8a80487db636d47cd408cf3a"> + Decreasing unnecessary wait time <a href="#b5c26b0e8a80487db636d47cd408cf3a">#</a> + </h3> + <p> + In the latter example, the 300 ms wait time for the parallel <code>PUT</code> requests are necessary to trigger the race condition, but the rest of the test's actions don't need slowing down. We can remove the unwarranted wait time by setting up two services: One slow, and one normal. + </p> + <p> + To be honest, I could have modelled this by just instantiating two service objects, but why do something as pedestrian as that when you can turn <code>RestaurantService</code> into a <a href="/2020/10/19/monomorphic-functors">monomorphic functor</a>? + </p> + <p> + <pre><span style="color:blue;">internal</span>&nbsp;<span style="color:#2b91af;">RestaurantService</span>&nbsp;<span style="font-weight:bold;color:#74531f;">Select</span>(<span style="color:#2b91af;">Func</span>&lt;<span style="color:#2b91af;">IReservationsRepository</span>,&nbsp;<span style="color:#2b91af;">IReservationsRepository</span>&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">selector</span>) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">if</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">selector</span>&nbsp;<span style="color:blue;">is</span>&nbsp;<span style="color:blue;">null</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">throw</span>&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">ArgumentNullException</span>(<span style="color:blue;">nameof</span>(<span style="font-weight:bold;color:#1f377f;">selector</span>)); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#8f08c4;">return</span>&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">RestaurantService</span>(<span style="font-weight:bold;color:#1f377f;">selector</span>(repository)); +}</pre> + </p> + <p> + Granted, this is verging on the frivolous, but when writing code for a blog post, I think I'm allowed a little fun. + </p> + <p> + In any case, this now enables me to rewrite the test like this: + </p> + <p> + <pre>[<span style="color:#2b91af;">Fact</span>] +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;<span style="color:#2b91af;">Task</span>&nbsp;<span style="font-weight:bold;color:#74531f;">NoOverbookingRace</span>() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">date</span>&nbsp;=&nbsp;<span style="color:#2b91af;">DateTime</span>.Now.Date.<span style="font-weight:bold;color:#74531f;">AddDays</span>(1).<span style="font-weight:bold;color:#74531f;">AddHours</span>(18.5); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">using</span>&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">RestaurantService</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">using</span>&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">slowService</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">from</span>&nbsp;repo&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">select</span>&nbsp;<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">SlowReservationsRepository</span>(<span style="color:#2b91af;">TimeSpan</span>.<span style="color:#74531f;">FromMilliseconds</span>(100),&nbsp;repo); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">task1</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">slowService</span>.<span style="font-weight:bold;color:#74531f;">PostReservation</span>(<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">ReservationDtoBuilder</span>() +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithDate</span>(<span style="font-weight:bold;color:#1f377f;">date</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithQuantity</span>(10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Build</span>()); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">task2</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">slowService</span>.<span style="font-weight:bold;color:#74531f;">PostReservation</span>(<span style="color:blue;">new</span>&nbsp;<span style="color:#2b91af;">ReservationDtoBuilder</span>() +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithDate</span>(<span style="font-weight:bold;color:#1f377f;">date</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">WithQuantity</span>(10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.<span style="font-weight:bold;color:#74531f;">Build</span>()); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">actual</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="color:#2b91af;">Task</span>.<span style="color:#74531f;">WhenAll</span>(<span style="font-weight:bold;color:#1f377f;">task1</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">task2</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">actual</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>.StatusCode&nbsp;==&nbsp;<span style="color:#2b91af;">HttpStatusCode</span>.InternalServerError); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">ok</span>&nbsp;=&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Single</span>(<span style="font-weight:bold;color:#1f377f;">actual</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>&nbsp;=&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">msg</span>.IsSuccessStatusCode); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Check&nbsp;that&nbsp;the&nbsp;reservation&nbsp;was&nbsp;actually&nbsp;created:</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">service</span>.<span style="font-weight:bold;color:#74531f;">GetReservation</span>(<span style="font-weight:bold;color:#1f377f;">ok</span>.Headers.Location); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">resp</span>.<span style="font-weight:bold;color:#74531f;">EnsureSuccessStatusCode</span>(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#8f08c4;">await</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">resp</span>.<span style="font-weight:bold;color:#74531f;">ParseJsonContent</span>&lt;<span style="color:#2b91af;">ReservationDto</span>&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Assert</span>.<span style="color:#74531f;">Equal</span>(10,&nbsp;<span style="font-weight:bold;color:#1f377f;">reservation</span>.Quantity); +}</pre> + </p> + <p> + Notice how only the parallel execution of <code>task1</code> and <code>task2</code> run on the slow system. The rest runs as fast as it can. It's as if the client was hitting two different servers that just happen to connect to the same database. Now the test only waits for the 200 ms described above. The <code>PUT</code> test, likewise, only idles for 300 ms instead of 900 ms. + </p> + <h3 id="90a4897a10b24d2c98d09129884f7e13"> + Near-deterministic tests <a href="#90a4897a10b24d2c98d09129884f7e13">#</a> + </h3> + <p> + Does this deterministically reproduce the race condition? In practice, it may move us close enough, but theoretically the race is still on. With the increased wait time, it's now much more unlikely that the race condition does <em>not</em> happen, but it still could. + </p> + <p> + Imagine that <code>task1</code> queries the Repository. Just as it's received a response, but before <code>task2</code> starts its query, execution is paused, perhaps because of garbage collection. Once the program resumes, <code>task1</code> runs to completion before <code>task2</code> reads from the database. In that case, <code>task2</code> ends up making the right decision, rejecting the reservation. Even if no transaction control were in place. + </p> + <p> + This may not be a particularly realistic scenario, but I suppose it could happen if the computer is stressed in general. Even so, you might decide to make such <a href="https://en.wikipedia.org/wiki/False_positives_and_false_negatives">false-negative</a> scenarios even more unlikely by increasing the delay time. Of course, the downside is that tests take even longer to run. + </p> + <p> + Another potential problem is that there's no <em>guarantee</em> that <code>task1</code> and <code>task2</code> run in parallel. Even if the test doesn't <code>await</code> any of the tasks, both start executing immediately. There's an (unlikely) chance that <code>task1</code> completes before <code>task2</code> starts. Again, I don't consider this likely, but I suppose it could happen because of thread starvation, generation 2 garbage collection, the disk running full, etc. The point is that the test shown here is still playing the odds, even if the odds are really good. + </p> + <h3 id="335493859fd0494488353e60de46c9c3"> + Conclusion <a href="#335493859fd0494488353e60de46c9c3">#</a> + </h3> + <p> + Instead of running a scenario 'enough' times that reproducing a race condition is likely, you can increase the odds to near-certainty by slowing down the race. In this example, the race involves a database, but you might also encounter race conditions internally in multi-threaded code. I'm not insisting that the technique described in this article applies universally, but if you can slow down certain interactions in the right way, you may be able reproduce problems as automated tests. + </p> + <p> + If you've ever troubleshot a race condition, you've probably tried inserting sleeps into the code in various places to understand the problem. As described above, a single, strategically-placed <code>Task.Delay</code> may be all you need to reproduce a problem. What escaped me for a long time, however, was that I didn't realize that I could cleanly insert such pauses into production code. Until my workshop participant suggested using a Decorator. + </p> + <p> + A delaying Decorator slows interactions with the database down sufficiently to reproduce the race condition as an automated test. + </p> +</div> + +<div id="comments"> + <hr> + <h2 id="comments-header"> + Comments + </h2> + <div class="comment" id="165739b939044dbc930c3f6185c40378"> + <div class="comment-author"><a href="https://medium.com/@nickkell">Nick</a> <a href="#165739b939044dbc930c3f6185c40378">#</a></div> + <div class="comment-content"> + <p> + Unfortunately I haven't read your book, so perhaps this question is already answered there: how would a transaction prevent a race condition? + I would have expected to solve it using optimistic concurrency control. + </p> + <p> + My other point is regarding artificial delays. How is that deterministic? If you're injecting decorators to take advantage of seams in the code, and you know the number of operations, then you don't need delays at all. + To highlight the race condition you need to ensure two requests have read the same data concurrently. After that you can allow the writes, expecting one to succeed and the other to fail. + You could use a synchronization primitive, such as CountdownEvent. Initialize it with a count of 2 to represent your concurrent requests. + Repository.ReadReservations() would call Signal() to decrement it after fetching from the database. Repository.Create() would call Wait() to ensure both reads had been made before writing to the database. + </p> + </div> + <div class="comment-date">2025-06-15 17:58 UTC</div> + </div> + + <div class="comment" id="b820ddd2766744f7be9ef61e3eb18714"> + <div class="comment-author"><a href="/">Mark Seemann</a> <a href="#b820ddd2766744f7be9ef61e3eb18714">#</a></div> + <div class="comment-content"> + <p> + Nick, thank you for writing. Regarding the transaction, as the above <code>TryCreate</code> snippet shows, it wraps both the read and the write operation, and since the default <a href="https://learn.microsoft.com/dotnet/api/system.transactions.isolationlevel">isolation level</a> is <code>Serializable</code>, + </p> + <blockquote> + "no new data can be added during the transaction." + </blockquote> + <p> + As to the second point, I don't think I've claimed that inducing an artificial delay as a Decorator is deterministic. To the contrary, I <a href="#90a4897a10b24d2c98d09129884f7e13">explicitly discussed how this is only near-deterministic</a>. + </p> + <p> + That said, you're correct that there's an even better way, and I have another article queued up that takes an approach similar to the one that you describe. + </p> + </div> + <div class="comment-date">2025-06-18 07:01 UTC</div> + </div> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/06/02/testing-races-with-a-slow-decorator + + + + Song recommendations as a Haskell Impureim Sandwich + https://blog.ploeh.dk/2025/05/26/song-recommendations-as-a-haskell-impureim-sandwich/ + Mon, 26 May 2025 07:15:00 UTC + + + +<div id="post"> + <p> + <em>A pure function on potentially massive data.</em> + </p> + <p> + This article is part of a larger article series called <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">Alternative ways to design with functional programming</a>. As the title suggests, these articles discuss various ways to apply functional-programming principles to a particular problem. All the articles engage with the same problem. In short, the task is to calculate song recommendations for a user, based on massive data sets. Earlier articles in this series give you detailed explanation of the problem. + </p> + <p> + In the <a href="/2025/05/19/song-recommendations-as-an-f-impureim-sandwich">previous article</a>, you saw how to refactor the 'base' <a href="https://fsharp.org/">F#</a> code base to a <a href="https://en.wikipedia.org/wiki/Pure_function">pure function</a>. In this article, you'll see the same refactoring applied to the 'base' <a href="https://www.haskell.org/">Haskell</a> code base shown in <a href="/2025/04/21/porting-song-recommendations-to-haskell">Porting song recommendations to Haskell</a>. + </p> + <p> + The Git branch discussed in this article is the <em>pure-function</em> branch in the Haskell Git repository. + </p> + <h3 id="0ed27ab7abea4417918789c4e30e1204"> + Collecting all the data <a href="#0ed27ab7abea4417918789c4e30e1204">#</a> + </h3> + <p> + Like in the previous articles, we may start by adding two more methods to the <code>SongService</code> type class, which will enable us to enumerate all songs and all users. The full type class, with all four methods, then looks like this: + </p> + <p> + <pre><span style="color:blue;">class</span>&nbsp;<span style="color:#2b91af;">SongService</span>&nbsp;a&nbsp;<span style="color:blue;">where</span> +&nbsp;&nbsp;<span style="color:#2b91af;">getAllSongs</span>&nbsp;<span style="color:blue;">::</span>&nbsp;a&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">Song</span>] +&nbsp;&nbsp;<span style="color:#2b91af;">getAllUsers</span>&nbsp;<span style="color:blue;">::</span>&nbsp;a&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">User</span>] +&nbsp;&nbsp;<span style="color:#2b91af;">getTopListeners</span>&nbsp;<span style="color:blue;">::</span>&nbsp;a&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">User</span>] +&nbsp;&nbsp;<span style="color:#2b91af;">getTopScrobbles</span>&nbsp;<span style="color:blue;">::</span>&nbsp;a&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">Scrobble</span>]</pre> + </p> + <p> + If you compare with the type class definition shown in the article <a href="/2025/04/21/porting-song-recommendations-to-haskell">Porting song recommendations to Haskell</a>, you'll see that <code>getAllSongs</code> and <code>getAllUsers</code> are the new methods. + </p> + <p> + They enable you to collect all top listeners, and all top scrobbles, even though it may take some time. To gather all the top listeners, we may add this <code>collectAllTopListeners</code> action: + </p> + <p> + <pre>collectAllTopListeners&nbsp;srvc&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;songs&nbsp;&lt;-&nbsp;getAllSongs&nbsp;srvc +&nbsp;&nbsp;listeners&nbsp;&lt;-&nbsp;newIORef&nbsp;Map.empty +&nbsp;&nbsp;forM_&nbsp;songs&nbsp;$&nbsp;\song&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;topListeners&nbsp;&lt;-&nbsp;getTopListeners&nbsp;srvc&nbsp;$&nbsp;songId&nbsp;song +&nbsp;&nbsp;&nbsp;&nbsp;modifyIORef&nbsp;listeners&nbsp;(Map.insert&nbsp;(songId&nbsp;song)&nbsp;topListeners) +&nbsp;&nbsp;readIORef&nbsp;listeners</pre> + </p> + <p> + Likewise, you can amass all the top scrobbles with a similar action: + </p> + <p> + <pre>collectAllTopScrobbles&nbsp;srvc&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;users&nbsp;&lt;-&nbsp;getAllUsers&nbsp;srvc +&nbsp;&nbsp;scrobbles&nbsp;&lt;-&nbsp;newIORef&nbsp;Map.empty +&nbsp;&nbsp;forM_&nbsp;users&nbsp;$&nbsp;\user&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;topScrobbles&nbsp;&lt;-&nbsp;getTopScrobbles&nbsp;srvc&nbsp;$&nbsp;userName&nbsp;user +&nbsp;&nbsp;&nbsp;&nbsp;modifyIORef&nbsp;scrobbles&nbsp;(Map.insert&nbsp;(userName&nbsp;user)&nbsp;topScrobbles) +&nbsp;&nbsp;readIORef&nbsp;scrobbles</pre> + </p> + <p> + As you may have noticed, they're so similar that, had there been <a href="https://en.wikipedia.org/wiki/Rule_of_three_(computer_programming)">more than two</a>, we might consider extracting the similar parts to a reusable operation. + </p> + <p> + In both cases, we start with the action that enables us to enumerate all the resources (songs or scrobbles) that we're interested in. For each of these, we then invoke the action to get the 'top' resources for that song or scrobble. There's a massive <em>n+1</em> problem here, but you could conceivably parallelize all these queries, as they're independent. Still, it's bound to take much time, possibly hours. + </p> + <p> + You may be wondering why I chose to use <code>IORef</code> values for both actions, instead of more <a href="/2015/08/03/idiomatic-or-idiosyncratic">idiomatic</a> combinator-based expressions. Indeed, that is what I would usually do, but in this case, these two actions are heavily IO-bound already, and it makes the Haskell code more similar to the F# code. Not that that is normally a goal, but here I thought it might help you, the reader, to compare the different code bases. + </p> + <p> + All the data is kept in a <a href="https://hackage.haskell.org/package/containers/docs/Data-Map-Strict.html">Map</a> per action, so two massive maps in all. Once these two actions return, we're done with the <em>read</em> phase of the <a href="/2025/01/13/recawr-sandwich">Recawr Sandwich</a>. + </p> + <h3 id="5378c29512584cab95d484b1bdd97fd9"> + Referentially transparent function with local mutation <a href="#5378c29512584cab95d484b1bdd97fd9">#</a> + </h3> + <p> + As a first step, we may wish to turn the <code>getRecommendations</code> action into a <a href="https://en.wikipedia.org/wiki/Referential_transparency">referentially transparent</a> function. If you look through the commits in the Git repository, you can see that I actually did this through a series of <a href="https://www.industriallogic.com/blog/whats-this-about-micro-commits/">micro-commits</a>, but here I only present a more coarse-grained version of the changes I made. + </p> + <p> + In this version, I've removed the <code>srvc</code> (<code>SongService</code>) parameter, and instead added the two maps <code>topScrobbles</code> and <code>topListeners</code>. + </p> + <p> + <pre><span style="color:#2b91af;">getRecommendations</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;[<span style="color:blue;">Scrobble</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;[<span style="color:blue;">User</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">Song</span>] +getRecommendations&nbsp;topScrobbles&nbsp;topListeners&nbsp;un&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;<span style="color:green;">--&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations +</span> +&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;scrobbles&nbsp;=&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;un&nbsp;topScrobbles +&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;scrobblesSnapshot&nbsp;=&nbsp;<span style="color:blue;">take</span>&nbsp;100&nbsp;$&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;scrobbleCount)&nbsp;scrobbles + +&nbsp;&nbsp;recommendationCandidates&nbsp;&lt;-&nbsp;newIORef&nbsp;<span style="color:blue;">[]</span> +&nbsp;&nbsp;forM_&nbsp;scrobblesSnapshot&nbsp;$&nbsp;\scrobble&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;otherListeners&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;(songId&nbsp;$&nbsp;scrobbledSong&nbsp;scrobble)&nbsp;topListeners +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;otherListenersSnapshot&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;20&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;userScrobbleCount)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;((10_000&nbsp;&lt;=)&nbsp;.&nbsp;userScrobbleCount)&nbsp;otherListeners + +&nbsp;&nbsp;&nbsp;&nbsp;forM_&nbsp;otherListenersSnapshot&nbsp;$&nbsp;\otherListener&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;otherScrobbles&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;(userName&nbsp;otherListener)&nbsp;topScrobbles +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;otherScrobblesSnapshot&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;10&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;songRating&nbsp;.&nbsp;scrobbledSong)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;(songHasVerifiedArtist&nbsp;.&nbsp;scrobbledSong)&nbsp;otherScrobbles + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;forM_&nbsp;otherScrobblesSnapshot&nbsp;$&nbsp;\otherScrobble&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;song&nbsp;=&nbsp;scrobbledSong&nbsp;otherScrobble +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;modifyIORef&nbsp;recommendationCandidates&nbsp;(song&nbsp;:) + +&nbsp;&nbsp;recommendations&nbsp;&lt;-&nbsp;readIORef&nbsp;recommendationCandidates +&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;$&nbsp;<span style="color:blue;">take</span>&nbsp;200&nbsp;$&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;songRating)&nbsp;recommendations</pre> + </p> + <p> + You've probably noticed that this action still looks impure, since it returns <code>IO [Song]</code>. Even so, it's referentially transparent, since it's fully deterministic and without side effects. + </p> + <p> + The way I refactored the action, this order of changes was what made most sense to me. Getting rid of the <code>SongService</code> parameter was more important to me than getting rid of the <code>IO</code> wrapper. + </p> + <p> + In any case, this is only an interim state towards a more idiomatic pure Haskell function. + </p> + <h3 id="5eba5b02d8724403b8ea6f26054fc9af"> + A single expression <a href="#5eba5b02d8724403b8ea6f26054fc9af">#</a> + </h3> + <p> + A curious property of expression-based languages is that you can conceivably write functions in 'one line of code'. Granted, it would often be a terribly wide line, not at all readable, a beast to maintain, and often with poor performance, so not something you'd want to alway do. + </p> + <p> + In this case, however, we <em>can</em> do that, although in order to stay within an <a href="/2019/11/04/the-80-24-rule">80x24 box</a>, we break the expression over multiple lines. + </p> + <p> + <pre><span style="color:#2b91af;">getRecommendations</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;[<span style="color:blue;">Scrobble</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;[<span style="color:blue;">User</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;[<span style="color:blue;">Song</span>] +getRecommendations&nbsp;topScrobbles&nbsp;topListeners&nbsp;un&nbsp;= +&nbsp;&nbsp;<span style="color:green;">--&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations +</span> +&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;200&nbsp;$ +&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;songRating)&nbsp;$ +&nbsp;&nbsp;<span style="color:blue;">fmap</span>&nbsp;scrobbledSong&nbsp;$ +&nbsp;&nbsp;(\otherListener&nbsp;-&gt;&nbsp;<span style="color:blue;">take</span>&nbsp;10&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;songRating&nbsp;.&nbsp;scrobbledSong)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;(songHasVerifiedArtist&nbsp;.&nbsp;scrobbledSong)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;(userName&nbsp;otherListener)&nbsp;topScrobbles)&nbsp;=&lt;&lt; +&nbsp;&nbsp;(\scrobble&nbsp;-&gt;&nbsp;<span style="color:blue;">take</span>&nbsp;20&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;userScrobbleCount)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;((10_000&nbsp;&lt;=)&nbsp;.&nbsp;userScrobbleCount)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;(songId&nbsp;$&nbsp;scrobbledSong&nbsp;scrobble)&nbsp;topListeners)&nbsp;=&lt;&lt; +&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;100 +&nbsp;&nbsp;(sortOn&nbsp;(Down&nbsp;.&nbsp;scrobbleCount)&nbsp;$&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;un&nbsp;topScrobbles)</pre> + </p> + <p> + This snapshot also got rid of the <code>IORef</code> value, and <code>IO</code> in general. The function is still referentially transparent, but now Haskell can also see that. + </p> + <p> + Even so, this looks dense and confusing. It doesn't help that Haskell should usually be read right-to-left, and bottom-to-top. Is it possible to improve the readability of this function? + </p> + <h3 id="c621ff8635bd4eabaa085b7b7d4a8c75"> + Composition from smaller functions <a href="#c621ff8635bd4eabaa085b7b7d4a8c75">#</a> + </h3> + <p> + To improve readability and maintainability, we may now extract helper functions. The first one easily suggests itself. + </p> + <p> + <pre><span style="color:#2b91af;">getUsersOwnTopScrobbles</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Ord</span>&nbsp;k&nbsp;<span style="color:blue;">=&gt;</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;k&nbsp;[<span style="color:blue;">Scrobble</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;k&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;[<span style="color:blue;">Scrobble</span>] +getUsersOwnTopScrobbles&nbsp;topScrobbles&nbsp;un&nbsp;= +&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;100&nbsp;$ +&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;scrobbleCount)&nbsp;$&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;un&nbsp;topScrobbles</pre> + </p> + <p> + Each of the subexpressions in the above code listing are candidates for the same kind of treatment, like this one: + </p> + <p> + <pre><span style="color:#2b91af;">getOtherUsersWhoListenedToTheSameSongs</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;[<span style="color:blue;">User</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">Scrobble</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;[<span style="color:blue;">User</span>] +getOtherUsersWhoListenedToTheSameSongs&nbsp;topListeners&nbsp;scrobble&nbsp;= +&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;20&nbsp;$ +&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;userScrobbleCount)&nbsp;$ +&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;((10_000&nbsp;&lt;=)&nbsp;.&nbsp;userScrobbleCount)&nbsp;$ +&nbsp;&nbsp;Map.findWithDefault&nbsp;<span style="color:blue;">[]</span>&nbsp;(songId&nbsp;$&nbsp;scrobbledSong&nbsp;scrobble)&nbsp;topListeners</pre> + </p> + <p> + You can't see it from the code listings themselves, but the module doesn't export these functions. They remain implementation details. + </p> + <p> + With a few more helper functions, you can now implement the <code>getRecommendations</code> function by composing the helpers. + </p> + <p> + <pre><span style="color:#2b91af;">getRecommendations</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;[<span style="color:blue;">Scrobble</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;[<span style="color:blue;">User</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;[<span style="color:blue;">Song</span>] +getRecommendations&nbsp;topScrobbles&nbsp;topListeners&nbsp;un&nbsp;= +&nbsp;&nbsp;<span style="color:green;">--&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations +</span> +&nbsp;&nbsp;aggregateSongsIntoRecommendations&nbsp;$ +&nbsp;&nbsp;getTopSongsOfOtherUser&nbsp;topScrobbles&nbsp;=&lt;&lt; +&nbsp;&nbsp;getOtherUsersWhoListenedToTheSameSongs&nbsp;topListeners&nbsp;=&lt;&lt; +&nbsp;&nbsp;getUsersOwnTopScrobbles&nbsp;topScrobbles&nbsp;un</pre> + </p> + <p> + Since Haskell by default composes from right to left, when you break such a composition over multiple lines (in order to stay within a <a href="/2019/11/04/the-80-24-rule">80x24</a> box), it should be read bottom-up. + </p> + <p> + You can remedy this situation by importing the <code>&</code> operator from <a href="https://hackage.haskell.org/package/base-4.21.0.0/docs/Data-Function.html">Data.Function</a>: + </p> + <p> + <pre><span style="color:#2b91af;">getRecommendations</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;[<span style="color:blue;">Scrobble</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">Map</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;[<span style="color:blue;">User</span>]&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;[<span style="color:blue;">Song</span>] +getRecommendations&nbsp;topScrobbles&nbsp;topListeners&nbsp;un&nbsp;=&nbsp; +&nbsp;&nbsp;getUsersOwnTopScrobbles&nbsp;topScrobbles&nbsp;un&nbsp;&gt;&gt;= +&nbsp;&nbsp;getOtherUsersWhoListenedToTheSameSongs&nbsp;topListeners&nbsp;&gt;&gt;= +&nbsp;&nbsp;getTopSongsOfOtherUser&nbsp;topScrobbles&nbsp;&amp; +&nbsp;&nbsp;aggregateSongsIntoRecommendations</pre> + </p> + <p> + Notice that I've named each of the helper functions after the code comments that accompanied the previous incarnations of this function. If we consider <a href="http://butunclebob.com/ArticleS.TimOttinger.ApologizeIncode">code comments apologies for not properly organizing the code</a>, we've now managed to structure it in such a way that those apologies are no longer required. + </p> + <h3 id="b934bca9ccf5403491d86a418d0ca8a3"> + Conclusion <a href="#b934bca9ccf5403491d86a418d0ca8a3">#</a> + </h3> + <p> + If you accept the (perhaps preposterous) assumption that it's possible to fit the required data in <a href="https://en.wikipedia.org/wiki/Persistent_data_structure">persistent data structures</a>, refactoring the recommendation algorithm to a pure function isn't that difficult. That's the pure part of a Recawr Sandwich. While I haven't shown the actual sandwich here, it's quite straightforward. You can find it in the tests in the Haskell Git repository. Also, once you've moved all the data collection to the boundary of the application, you may no longer need the <code>SongService</code> type class. + </p> + <p> + I find the final incarnation of the code shown here to be quite attractive. While I've kept the helper functions private to the module, it's always an option to export them if you find that warranted. This could improve testability of the overall code base, albeit at the risk of increasing the surface area of the API that you have to maintain and secure. + </p> + <p> + There are always trade-offs to be considered. Even if you, eventually, find that for this particular example, the input data size is just <em>too</em> big to make this alternative viable, there are, in my experience, many other situations when this kind of architecture is a good solution. Even if the input size is a decent amount of megabytes, the simplification offered by an <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> may trump the larger memory footprint. As always, if you're concerned about performance, <a href="https://ericlippert.com/2012/12/17/performance-rant/">measure it</a>. + </p> + <p> + This article concludes the overview of using an Recawr Sandwich to address the problem. Since it's, admittedly, a bit of a stretch to imagine running a program that uses terabytes (or more) of memory, we now turn to alternative architectures. + </p> + <p> + <strong>Next:</strong> <a href="/2025/06/09/song-recommendations-from-combinators">Song recommendations from combinators</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/05/26/song-recommendations-as-a-haskell-impureim-sandwich + + + + Song recommendations as an F# Impureim Sandwich + https://blog.ploeh.dk/2025/05/19/song-recommendations-as-an-f-impureim-sandwich/ + Mon, 19 May 2025 08:06:00 UTC + + + +<div id="post"> + <p> + <em>A pure function on potentially massive data.</em> + </p> + <p> + This article is part of a larger article series titled <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">Alternative ways to design with functional programming</a>. In the <a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">previous article</a>, you saw an example of applying the <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> pattern to the problem at hand: A song recommendation engine that sifts through much historical data. + </p> + <p> + As already covered in <a href="/2025/04/28/song-recommendations-as-an-impureim-sandwich">Song recommendations as an Impureim Sandwich</a>, the drawback, if you will, of a <a href="/2025/01/13/recawr-sandwich">Recawr Sandwich</a> is that you need to collect all data from impure sources before you can pass it to a <a href="https://en.wikipedia.org/wiki/Pure_function">pure function</a>. It may happen that you need so much data that this strategy becomes untenable. <a href="/2025/05/12/song-recommendations-proof-of-concept-memory-measurements">This may be the case here</a>, but surprisingly often, what strikes us humans as being vast amounts are peanuts for computers. + </p> + <p> + So even if you don't find this particular example realistic, I'll forge ahead and show how to apply the Recawr Sandwich pattern to this problem. This is essentially a port to <a href="https://fsharp.org/">F#</a> of the C# code from the previous article. If you rather want to see some more realistic architectures to deal with the overall problem, you can always go back to the table of contents in the <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">first article of the series</a>. + </p> + <p> + In this article, I'm working with the <em>fsharp-pure-function</em> branch of the Git repository. + </p> + <h3 id="090279ba52c044c1b9a38d6d29ce26f1"> + Collecting all the data <a href="#090279ba52c044c1b9a38d6d29ce26f1">#</a> + </h3> + <p> + Like in the previous article, we may start by adding two more members to the <code>SongService</code> interface, which will enable us to enumerate all songs and all users. The full interface, with all four methods, then looks like this: + </p> + <p> + <pre><span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">SongService</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">abstract</span>&nbsp;<span style="font-weight:bold;color:#74531f;">GetAllSongs</span>&nbsp;:&nbsp;<span style="color:#2b91af;">unit</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IEnumerable</span>&lt;<span style="color:#2b91af;">Song</span>&gt;&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">abstract</span>&nbsp;<span style="font-weight:bold;color:#74531f;">GetAllUsers</span>&nbsp;:&nbsp;<span style="color:#2b91af;">unit</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IEnumerable</span>&lt;<span style="color:#2b91af;">User</span>&gt;&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">abstract</span>&nbsp;<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>&nbsp;:&nbsp;songId&nbsp;:&nbsp;<span style="color:#2b91af;">int</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">User</span>&gt;&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">abstract</span>&nbsp;<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>&nbsp;:&nbsp;userName&nbsp;:&nbsp;<span style="color:#2b91af;">string</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&gt;</pre> + </p> + <p> + If you compare with the interface definition shown in the article <a href="/2025/04/14/porting-song-recommendations-to-f">Porting song recommendations to F#</a>, you'll see that <code>GetAllSongs</code> and <code>GetAllUsers</code> are the new methods. + </p> + <p> + They enable you to collect all top listeners, and all top scrobbles, even though it may take some time. To gather all the top listeners, we may add this <code>collectAllTopListeners</code> action: + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">collectAllTopListeners</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">songService</span>&nbsp;:&nbsp;<span style="color:#2b91af;">SongService</span>)&nbsp;=&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>&nbsp;=&nbsp;<span style="color:#2b91af;">Dictionary</span>&lt;<span style="color:#2b91af;">int</span>,&nbsp;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">User</span>&gt;&gt;&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetAllSongs</span>&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">do!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">TaskSeq</span>.<span style="color:#74531f;">iter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Id +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>.<span style="font-weight:bold;color:#74531f;">Add</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">s</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>)&nbsp;}&nbsp;) +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>&nbsp;:&gt;&nbsp;<span style="color:#2b91af;">IReadOnlyDictionary</span>&lt;_,&nbsp;_&gt;&nbsp;}</pre> + </p> + <p> + Likewise, you can amass all the top scrobbles with a similar action: + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">collectAllTopScrobbles</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">songService</span>&nbsp;:&nbsp;<span style="color:#2b91af;">SongService</span>)&nbsp;=&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>&nbsp;=&nbsp;<span style="color:#2b91af;">Dictionary</span>&lt;<span style="color:#2b91af;">string</span>,&nbsp;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&gt;&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetAllUsers</span>&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">do!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">TaskSeq</span>.<span style="color:#74531f;">iter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.UserName +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>.<span style="font-weight:bold;color:#74531f;">Add</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">u</span>.UserName,&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>)&nbsp;}&nbsp;) +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>&nbsp;:&gt;&nbsp;<span style="color:#2b91af;">IReadOnlyDictionary</span>&lt;_,&nbsp;_&gt;&nbsp;}</pre> + </p> + <p> + As you may have noticed, they're so similar that, had there been <a href="https://en.wikipedia.org/wiki/Rule_of_three_(computer_programming)">more than two</a>, we might consider extracting the similar parts to a reusable operation. + </p> + <p> + In both cases, we start with the action that enables us to enumerate all the resources (songs or scrobbles) that we're interested in. For each of these, we then invoke the action to get the 'top' resources for that song or scrobble. There's a massive <em>n+1</em> problem here, but you could conceivably parallelize all these queries, as they're independent. Still, it's bound to take much time, possibly hours. + </p> + <p> + All the data is kept in a dictionary per action, so two massive dictionaries in all. Once these two actions return, we're done with the <em>read</em> phase of the Recawr Sandwich. + </p> + <h3 id="5194a0ec946046068d42c086f7fd2697"> + Traversals <a href="#5194a0ec946046068d42c086f7fd2697">#</a> + </h3> + <p> + You may have wondered about the above <code>TaskSeq.iter</code> action. That's not part of the standard library. What is it, and where does it come from? + </p> + <p> + It's a specialized <a href="/2024/11/11/traversals">traversal</a>, designed to make asynchronous <a href="https://en.wikipedia.org/wiki/Command%E2%80%93query_separation">Commands</a> more streamlined. + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">iter</span>&nbsp;<span style="color:#74531f;">f</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">xs</span>&nbsp;=&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">units</span>&nbsp;=&nbsp;<span style="color:#74531f;">traverse</span>&nbsp;<span style="color:#74531f;">f</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">xs</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">iter</span>&nbsp;<span style="color:#74531f;">id</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">units</span>&nbsp;}</pre> + </p> + <p> + If you've ever wondered why the <em>identity function</em> (<code>id</code>) is useful, here's an example. In the first line of code, <code>units</code> is a <code>unit seq</code> value; i.e. a sequence of <code>unit</code> values. To make <code>TaskSeq.iter</code> as easy to use as possible, it should turn that multitude of <code>unit</code> values into a single <code>unit</code> value. There's more than one way to do that, but I found that using <code>Seq.iter</code> was about the most succinct option I could think of. Be that as it may, <code>Seq.iter</code> requires as an argument a function that returns <code>unit</code>, and since we already have <code>unit</code> values, <code>id</code> does the job. + </p> + <p> + The <code>iter</code> action uses the <code>TaskSeq</code> module's <code>traverse</code> function, which is defined like this: + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">traverse</span>&nbsp;<span style="color:#74531f;">f</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">xs</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">go</span>&nbsp;<span style="color:#1f377f;">acc</span>&nbsp;<span style="color:#1f377f;">x</span>&nbsp;=&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">x&#39;</span>&nbsp;=&nbsp;<span style="color:#1f377f;">x</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">acc&#39;</span>&nbsp;=&nbsp;<span style="color:#1f377f;">acc</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">append</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">acc&#39;</span>&nbsp;[<span style="font-weight:bold;color:#1f377f;">x&#39;</span>]&nbsp;} +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">xs</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">map</span>&nbsp;<span style="color:#74531f;">f</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">fold</span>&nbsp;<span style="color:#74531f;">go</span>&nbsp;(<span style="color:blue;">task</span>&nbsp;{&nbsp;<span style="color:blue;">return</span>&nbsp;[]&nbsp;})</pre> + </p> + <p> + The type of <code>traverse</code> is <code>(&#39;a&nbsp;-&gt;&nbsp;#Task&lt;&#39;c&gt;)&nbsp;-&gt;&nbsp;&#39;a&nbsp;seq&nbsp;-&gt;&nbsp;Task&lt;&#39;c&nbsp;seq&gt;</code>; that is, it applies an asynchronous action to each of a sequence of <code>'a</code> values, and returns an asynchronous workflow that contains a sequence of <code>'c</code> values. + </p> + <h3 id="16c034285a8d41b39923e27c6b81788e"> + Dictionary lookups <a href="#16c034285a8d41b39923e27c6b81788e">#</a> + </h3> + <p> + In .NET, queries that may fail are <a href="/2015/08/03/idiomatic-or-idiosyncratic">idiomatically</a> modelled with methods that take <code>out</code> parameters. This is also true for dictionary lookups. Since that kind of design doesn't compose well, it's useful to add a little helper function that instead may return an empty value. While you'd <a href="/2019/07/15/tester-doer-isomorphisms">generally do that by returning an option value</a>, in this case, an empty collection is more appropriate. + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">key</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">d</span>&nbsp;:&nbsp;<span style="color:#2b91af;">IReadOnlyDictionary</span>&lt;_,&nbsp;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;_&gt;&gt;)&nbsp;= +&nbsp;&nbsp;&nbsp;<span style="color:blue;">match</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">d</span>.<span style="font-weight:bold;color:#74531f;">TryGetValue</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">key</span>&nbsp;<span style="color:blue;">with</span> +&nbsp;&nbsp;&nbsp;|&nbsp;<span style="color:blue;">true</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">v</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">v</span> +&nbsp;&nbsp;&nbsp;|&nbsp;_&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">List</span>.empty</pre> + </p> + <p> + You may have noticed that I also added a similar helper function in <a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">the C# example</a>, although there I called it <code>GetOrEmpty</code>. + </p> + <h3 id="cddd7cc72cf748b995fc0b58b0971c78"> + Pure function with local mutation <a href="#cddd7cc72cf748b995fc0b58b0971c78">#</a> + </h3> + <p> + As a first step, we may wish to turn the <code>GetRecommendationsAsync</code> method into a pure function. If you look through the commits in the Git repository, you can see that I actually did this through a series of <a href="https://www.industriallogic.com/blog/whats-this-about-micro-commits/">micro-commits</a>, but here I only present a more coarse-grained version of the changes I made. + </p> + <p> + Instead of a method on a class, we now have a self-contained function that takes, as arguments, two dictionaries, but no <code>SongService</code> dependency. + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">getRecommendations</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobblesSnapshot</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.ScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;100 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>&nbsp;=&nbsp;<span style="color:#2b91af;">ResizeArray</span>&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">for</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobblesSnapshot</span>&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListeners</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>.Song.Id +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListenersSnapshot</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListeners</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount&nbsp;&gt;=&nbsp;10_000) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;20 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">for</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListener</span>&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListenersSnapshot</span>&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListener</span>.UserName +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobblesSnapshot</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.IsVerifiedArtist) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;10 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobblesSnapshot</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">List</span>.<span style="color:#74531f;">map</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>.<span style="font-weight:bold;color:#74531f;">AddRange</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendations</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;200 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;:&gt;&nbsp;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;_&gt; + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendations</span></pre> + </p> + <p> + Since this is now a pure function, there's no need to run as an asynchronous workflow. The function no longer returns a <code>Task</code>, and I've also dispensed with the <em>Async</em> suffix. + </p> + <p> + The implementation still has imperative remnants. It initializes an empty <code>ResizeArray</code> (AKA <code>List&lt;T&gt;</code>), and loops through nested loops to repeatedly call <a href="https://learn.microsoft.com/dotnet/api/system.collections.generic.list-1.addrange">AddRange</a>. + </p> + <p> + Even though the function contains local state mutation, none of it escapes the function's scope. The function is <a href="https://en.wikipedia.org/wiki/Referential_transparency">referentially transparent</a> because it always returns the same result when given the same input, and it has no side effects. + </p> + <p> + You might still wish that it was 'more functional', which is certainly possible. + </p> + <h3 id="dda1233fb2ff4bc3b21d49d910fabf6f"> + A single expression <a href="#dda1233fb2ff4bc3b21d49d910fabf6f">#</a> + </h3> + <p> + A curious property of expression-based languages is that you can conceivably write functions in 'one line of code'. Granted, it would often be a terribly wide line, not at all readable, a beast to maintain, and often with poor performance, so not something you'd want to alway do. + </p> + <p> + In this case, however, we <em>can</em> do that, although in order to stay within an <a href="/2019/11/04/the-80-24-rule">80x24 box</a>, we break the expression over multiple lines. + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">getRecommendations</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span> +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.ScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;100 +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">collect</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>&nbsp;<span style="color:blue;">-&gt;</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>.Song.Id +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount&nbsp;&gt;=&nbsp;10_000) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;20 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">collect</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListener</span>&nbsp;<span style="color:blue;">-&gt;</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListener</span>.UserName +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.IsVerifiedArtist) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;10 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">map</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song))) +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;200 +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> +&nbsp;&nbsp;&nbsp;&nbsp;:&gt;&nbsp;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;_&gt;</pre> + </p> + <p> + To be honest, the four lines of comments push the function definition over the edge of 24 lines of code, but without them, this variation actually does fit an 80x24 box. Even so, I'm not arguing that this is the best possible way to organize and lay out this function. + </p> + <p> + You may rightly complain that it's too dense. Perhaps you're also concerned about the <a href="https://wiki.c2.com/?ArrowAntiPattern">arrow code</a> tendency. + </p> + <p> + I'm not disagreeing, but at least this represents a milestone where the function is not only referentially transparent, but also implemented without local mutation. Not that that really should be the most important criterion, but once you have an entirely expression-based implementation, it's usually easier to break it up into smaller building blocks. + </p> + <h3 id="7d81ce38521b406fa306d4bfa79a44bb"> + Composition from smaller functions <a href="#7d81ce38521b406fa306d4bfa79a44bb">#</a> + </h3> + <p> + To improve readability and maintainability, we may now extract helper functions. The first one easily suggests itself. + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:#74531f;">getUsersOwnTopScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span> +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.ScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;100</pre> + </p> + <p> + Each of the subexpressions in the above code listing are candidates for the same kind of treatment, like this one: + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:#74531f;">getOtherUsersWhoListenedToTheSameSongs</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span> +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Dict</span>.<span style="color:#74531f;">findOrEmpty</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>.Song.Id +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;-&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount&nbsp;&gt;=&nbsp;10_000) +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;-&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;20</pre> + </p> + <p> + Notice that these helper methods are marked <code>private</code> so that they remain implementation details within the module that exports the <code>getRecommendations</code> function. + </p> + <p> + With a few more helper functions, you can now implement the <code>getRecommendations</code> function by composing the helpers. + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">getRecommendations</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#74531f;">getUsersOwnTopScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&gt;&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">collect</span>&nbsp;( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#74531f;">getOtherUsersWhoListenedToTheSameSongs</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topListeners</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&gt;&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">collect</span>&nbsp;(<span style="color:#74531f;">getTopSongsOfOtherUser</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">topScrobbles</span>)) +&nbsp;&nbsp;&nbsp;&nbsp;&gt;&gt;&nbsp;<span style="color:#74531f;">aggregateSongsIntoRecommendations</span></pre> + </p> + <p> + Notice that I've named each of the helper functions after the code comments that accompanied the previous incarnations of this function. If we consider <a href="http://butunclebob.com/ArticleS.TimOttinger.ApologizeIncode">code comments apologies for not properly organizing the code</a>, we've now managed to structure it in such a way that those apologies are no longer required. + </p> + <h3 id="7467fbb8e803446ea3dff035e7388301"> + Conclusion <a href="#7467fbb8e803446ea3dff035e7388301">#</a> + </h3> + <p> + If you accept the (perhaps preposterous) assumption that it's possible to fit the required data in <a href="https://en.wikipedia.org/wiki/Persistent_data_structure">persistent data structures</a>, refactoring the recommendation algorithm to a pure function isn't that difficult. That's the pure part of a Recawr Sandwich. While I haven't shown the actual sandwich here, it's identical to the example shown in <a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">Song recommendations as a C# Impureim Sandwich</a>. + </p> + <p> + I find the final incarnation of the code shown here to be quite attractive. While I've kept the helper functions <code>private</code>, it's always an option to promote them to <code>public</code> functions if you find that warranted. This could improve testability of the overall code base, albeit at the risk of increasing the surface area of the API that you have to maintain and secure. + </p> + <p> + There are always trade-offs to be considered. Even if you, eventually, find that for this particular example, the input data size is just <em>too</em> big to make this alternative viable, there are, in my experience, many other situations when this kind of architecture is a good solution. Even if the input size is a decent amount of megabytes, the simplification offered by an Impureim Sandwich may trump the larger memory footprint. As always, if you're concerned about performance, <a href="https://ericlippert.com/2012/12/17/performance-rant/">measure it</a>. + </p> + <p> + Before we turn to alternative architectures, we'll survey how this variation looks in <a href="https://www.haskell.org/">Haskell</a>. As is generally the case in this article series, if you don't care about Haskell, you can always go back to the table of contents in the <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">first article in the series</a> and instead navigate to the next article that interests you. + </p> + <p> + <strong>Next:</strong> <a href="/2025/05/26/song-recommendations-as-a-haskell-impureim-sandwich">Song recommendations as a Haskell Impureim Sandwich</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/05/19/song-recommendations-as-an-f-impureim-sandwich + + + + Song recommendations proof-of-concept memory measurements + https://blog.ploeh.dk/2025/05/12/song-recommendations-proof-of-concept-memory-measurements/ + Mon, 12 May 2025 07:52:00 UTC + + + +<div id="post"> + <p> + <em>An attempt at measurement, and some results.</em> + </p> + <p> + This is an article in a <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">larger series about functional programming design alternatives</a>, and a direct continuation of the <a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">previous article</a>. The question lingering after the <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> proof of concept is: What are the memory requirements of front-loading all users, songs, and scrobbles? + </p> + <p> + One can guess, as I've <a href="/2025/04/28/song-recommendations-as-an-impureim-sandwich">already done</a>, but it's safer to measure. In this article, you'll find a description of the experiment, as well as some results. + </p> + <h3 id="6f88f7533af54936a2a923903171c831"> + Test program <a href="#6f88f7533af54936a2a923903171c831">#</a> + </h3> + <p> + Since I don't measure application memory profiles that often, I searched the web to learn how, and found <a href="https://stackoverflow.com/a/28514434/126014">this answer</a> by <a href="https://stackoverflow.com/users/22656/jon-skeet">Jon Skeet</a>. That's a reputable source, so I'm assuming that the described approach is appropriate. + </p> + <p> + I added a new command-line executable to the source code and made this the entry point: + </p> + <p> + <pre><span style="color:blue;">const</span>&nbsp;<span style="color:blue;">int</span>&nbsp;size&nbsp;=&nbsp;100_000; + +<span style="color:blue;">static</span>&nbsp;<span style="color:blue;">async</span>&nbsp;Task&nbsp;Main() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;before&nbsp;=&nbsp;GC.GetTotalMemory(<span style="color:blue;">true</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;var&nbsp;(listeners,&nbsp;scrobbles)&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;Populate(); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;after&nbsp;=&nbsp;GC.GetTotalMemory(<span style="color:blue;">true</span>); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;diff&nbsp;=&nbsp;after&nbsp;-&nbsp;before; + +&nbsp;&nbsp;&nbsp;&nbsp;Console.WriteLine(<span style="color:#a31515;">&quot;Total&nbsp;memory:&nbsp;{0:N0}B.&quot;</span>,&nbsp;diff); + +&nbsp;&nbsp;&nbsp;&nbsp;GC.KeepAlive(listeners); +&nbsp;&nbsp;&nbsp;&nbsp;GC.KeepAlive(scrobbles); +}</pre> + </p> + <p> + <code>listeners</code> and <code>scrobbles</code> are two dictionaries of data, as described in the <a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">previous article</a>. Together, they contain the data that we measure. Both are populated by this method: + </p> + <p> + <pre><span style="color:blue;">private</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:blue;">async</span>&nbsp;Task&lt;( +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;)&gt;&nbsp;Populate() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;service&nbsp;=&nbsp;PopulateService(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;listeners&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;service.CollectAllTopListeners(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;scrobbles&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;service.CollectAllTopScrobbles(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;(listeners,&nbsp;scrobbles); +}</pre> + </p> + <p> + The <code>service</code> variable is a <code>FakeSongService</code> object populated with randomly generated data. The <code>CollectAllTopListeners</code> and <code>CollectAllTopScrobbles</code> methods are the same as described in the previous article. When the method returns the two dictionaries, the <code>service</code> object goes out of scope and can be garbage-collected. When the program measures the memory load, it measures the size of the two dictionaries, but not <code>service</code>. + </p> + <p> + I've reused the <a href="https://fscheck.github.io/FsCheck/">FsCheck</a> generators for random data generation: + </p> + <p> + <pre><span style="color:blue;">private</span>&nbsp;<span style="color:blue;">static</span>&nbsp;SongService&nbsp;PopulateService() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;users&nbsp;=&nbsp;RecommendationsProviderTests.Gen.UserName.Sample(size); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;songs&nbsp;=&nbsp;RecommendationsProviderTests.Gen.Song.Sample(size); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;scrobbleGen&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">from</span>&nbsp;user&nbsp;<span style="color:blue;">in</span>&nbsp;Gen.Elements(users) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">from</span>&nbsp;song&nbsp;<span style="color:blue;">in</span>&nbsp;Gen.Elements(songs) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">from</span>&nbsp;scrobbleCount&nbsp;<span style="color:blue;">in</span>&nbsp;Gen.Choose(1,&nbsp;10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">select</span>&nbsp;(user,&nbsp;song,&nbsp;scrobbleCount); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;service&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;FakeSongService(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">foreach</span>&nbsp;(var&nbsp;(user,&nbsp;song,&nbsp;scrobbleCount)&nbsp;<span style="color:blue;">in</span>&nbsp;scrobbleGen.Sample(size)) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;service.Scrobble(user,&nbsp;song,&nbsp;scrobbleCount); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;service; +}</pre> + </p> + <p> + A <code>Gen&lt;T&gt;</code> object comes with a <code>Sample</code> method you can use to request a specified number of randomly generated values. + </p> + <p> + In order to keep the code simple, I used the <code>size</code> value for both the number of songs, number of users, and number of scrobbles. This probably creates too few scrobbles; a topic that requires further discussion later. + </p> + <h3 id="2759c45eda984c02871d99ff21d9936d"> + Measurements <a href="#2759c45eda984c02871d99ff21d9936d">#</a> + </h3> + <p> + I ran the above program with various <code>size</code> values; <em>100,000</em> up to <em>1,000,000</em> in <em>100,000</em> increments, and from there up to <em>1,000,000</em> (one million) in <em>500,000</em> increments. At the higher values, it took a good ten minutes to run the program. + </p> + <p> + <img src="/content/binary/song-recommendations-memory-costs.png" alt="Song recommendations memory cost line chart."> + </p> + <p> + As the chart indicates, I ran the program with various data representations (more on that below). While there are four distinct data series, they overlap pairwise so perfectly that the graph doesn't show the difference. The <em>record</em> and <em>struct record</em> data series are so identical that you can't visibly see the difference. The same is true for the <em>bitmasked class</em> and the <em>bitmasked struct</em> data series, which only go to <code>size</code> <em>500,000</em>. + </p> + <p> + There are small, but noticeable jumps from <em>4,500,000</em> to <em>5,000,000</em> and again from <em>8,500,000</em> to <em>9,000,000</em>, but the overall impression is that the relationship is linear. It seems safe to conclude that the solution scales linearly with the data size. + </p> + <p> + The number of bytes per size is almost constant and averages to 178 bytes. How does that compare to <a href="/2025/04/28/song-recommendations-as-an-impureim-sandwich">my previous memory size estimates</a>? There, I estimated a song and a scrobble to require 8 bytes each, and a user less than 32 bytes. The way the above simulation runs, it generates one song, one user, and one scrobble per size unit. Therefore, I'd expect the average memory cost per experiment size to be around <em>8 + 8 + 32 = 48</em>, plus some overhead from the dictionaries. + </p> + <p> + Given that the number I measure is 178, that's 130 bytes of overhead. Honestly, that's more than I expected. I expect a dictionary to maintain an array of keys, perhaps hashed with a bucket per hash value. Perhaps, had I picked another data structure than a plain old <a href="https://learn.microsoft.com/dotnet/api/system.collections.generic.dictionary-2">Dictionary</a>, it's possible that the overhead would be different. Or perhaps I just don't understand .NET's memory model, when push comes to shove. + </p> + <p> + I then tried to split the single <code>size</code> parameter into three that would control the number of users, songs, and scrobbles independently. Setting both the number of users and songs to ten million, I then ran a series of simulations with increasing scrobble counts. + </p> + <p> + <img src="/content/binary/song-recommendations-scrobble-memory-costs.png" alt="Scrobble memory cost line chart."> + </p> + <p> + The relationship still looks linear, and at a hundred million scrobbles (and ten million users and ten million songs), the simulation uses 8.3 GB of memory. + </p> + <p> + I admit that I'm still a bit puzzled by the measurements, compared to my previous estimates. I would have expected those sizes to require about 1,2 GB, plus overhead, so the actual measurements are off by a factor of 7. Not quite an order of magnitude, but close. + </p> + <h3 id="b1a0539a274f49ef937b15fc07090990"> + Realism <a href="#b1a0539a274f49ef937b15fc07090990">#</a> + </h3> + <p> + How useful are these measurements? How realistic are the experiments' parameters? Most streaming audio services report having catalogues with around 100 million songs, which is ten times more than what I've measured here. Such services may also have significantly more users than ten million, but what is going to make or break this architecture option (keeping all data in memory) is how many scrobbles users have, and how many times they listen to each song. + </p> + <p> + Even if we naively still believe that a scrobble only takes up 8 bytes, it doesn't follow automatically that 100 scrobbles take up 800 bytes. It depends on how many repeats there are. Recall how we may model a scrobble: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">sealed</span>&nbsp;<span style="color:blue;">record</span>&nbsp;<span style="color:#2b91af;">Scrobble</span>(Song&nbsp;<span style="font-weight:bold;color:#1f377f;">Song</span>,&nbsp;<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">ScrobbleCount</span>);</pre> + </p> + <p> + If a user listens to the same song ten times, we don't have to create ten <code>Scrobble</code> objects; we can create one and set the <code>ScrobbleCount</code> to <code>10</code>. + </p> + <p> + The memory requirement to store users' scrobbles depend on the average listening pattern. Even with millions of users, we may be able to store scrobbles in memory if users listen to relatively few songs. On the other hand, if they only listen to each song once, it's probably not going to fit in memory. + </p> + <p> + Still, we're dangerously close to the edge of what we can fit in memory. Shouldn't I just declare bankruptcy on that idea and move on? + </p> + <p> + The purpose of this <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">overall article series</a> is to demonstrate <em>alternatives</em> to the <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> pattern, so I'm ultimately going to do exactly that: Move on. + </p> + <p> + But not yet. + </p> + <h3 id="a5fce68a31c7414386c451ea2fe219be"> + Sharding <a href="#a5fce68a31c7414386c451ea2fe219be">#</a> + </h3> + <p> + Some applications are truly global in nature, and when that's the case, keeping everything in memory may not be 'web scale'. + </p> + <p> + Still, I've seen more than one international company treat geographic areas as separate entities. This may be for legal reasons, or other business concerns that are unrelated to technology constraints. + </p> + <p> + As a programmer, you may think that a song recommendations service ought to be truly global. After all, more data produces more accurate results, right? + </p> + <p> + Your business owners may not think so. They may be concerned that regional music tastes may 'bleed over' market boundaries, and that this could ultimately scare customers away. + </p> + <p> + Even if you can technically prove that this isn't a relevant concern, because you can write an algorithm that takes this into account, you may get a direct order that, say, Southeast Asian scrobbles may not be used in North America, or vice verse. + </p> + <p> + It's worth investigating whether such business or legal constraints are already in place, because if they are, this may mean that you can shard the data, and that each shard still fits in memory. + </p> + <p> + You may still think that I'm trying to salvage a bad idea, but that's not my agenda. I discuss these topics because in my experience, many programmers don't consider them. Understanding the wider context of a problem may suggest solutions that you might otherwise dismiss. + </p> + <p> + But what if the business constraints change in the future? Shouldn't we be ready for that? + </p> + <p> + Yes and no. You should consider how such changes would impact the architecture. Then you discuss the advantages and disadvantages with other stakeholders. + </p> + <p> + Keep in mind that the reason to consider an Impureim Sandwich is because it's <em>simple</em> and easy to implement and maintain. Other alternatives may be more 'scalable', but also riskier to implement. You should involve other stakeholders in such decisions. + </p> + <h3 id="2dee0af02fbb4ac9b9176685ecb2cc3a"> + Song representations <a href="#2dee0af02fbb4ac9b9176685ecb2cc3a">#</a> + </h3> + <p> + The first of the above charts draws graphs for four data series: + </p> + <ul> + <li>struct record</li> + <li>record</li> + <li>bitmasked struct</li> + <li>bitmasked class</li> + </ul> + <p> + These measure four different ways to model data; here more specifically a song. + </p> + <p> + My initial model of a song was a <code>record</code>: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">sealed</span>&nbsp;<span style="color:blue;">record</span>&nbsp;<span style="color:#2b91af;">Song</span>(<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">Id</span>,&nbsp;<span style="color:blue;">bool</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">IsVerifiedArtist</span>,&nbsp;<span style="color:blue;">byte</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">Rating</span>);</pre> + </p> + <p> + Then I thought that perhaps, since the type only contains <a href="https://learn.microsoft.com/dotnet/csharp/language-reference/builtin-types/value-types">value types</a>, it might be better to turn the above <code>record</code> into a <code>record struct</code>: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">record</span>&nbsp;<span style="color:blue;">struct</span>&nbsp;<span style="color:#2b91af;">Song</span>(<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">Id</span>,&nbsp;<span style="color:blue;">bool</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">IsVerifiedArtist</span>,&nbsp;<span style="color:blue;">byte</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">Rating</span>);</pre> + </p> + <p> + It turns out that it makes no visible difference. In the chart, the two data series are so close to each other that you can't distinguish them. + </p> + <p> + Then I thought that instead of an <code>int</code>, a <code>bool</code>, and a <code>byte</code>, I could use a single bitmask to model all three values. + </p> + <p> + After all, I was only guessing when it came to data types anyway. It's likely that <code>Rating</code> is only a five-point or ten-point scale, but I still used a <code>byte</code> to model it. This suggests that I'm not using 96% of the data type's range. Perhaps I could use one of those redundant bits for <code>IsVerifiedArtist</code>, instead of an entire <code>bool</code>. + </p> + <p> + Taking this further, modelling the <code>Id</code> as an <code>int</code> suggests that you may have 4,294,967,295 unique songs. That's 4.3 <em>billion</em> songs - at least an order of magnitude more than those 100 million songs that we hear about. In reality though, most systems that use <code>int</code> for IDs only do so because <code>int</code> is CLS-compliant, and <a href="https://learn.microsoft.com/dotnet/api/system.uint32">uint is not</a>. In other words, most systems that use <code>int</code> for IDs most likely only use the positive half, which means there are 16 bytes to use for other purposes. Enter the <em>bitmasked song:</em> + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">readonly</span>&nbsp;<span style="color:blue;">struct</span>&nbsp;<span style="color:#2b91af;">Song</span> +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:blue;">const</span>&nbsp;<span style="color:blue;">uint</span>&nbsp;idMask&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0b0000_0111_1111_1111_1111_1111_1111_1111; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:blue;">const</span>&nbsp;<span style="color:blue;">uint</span>&nbsp;isVerifiedArtistMask&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0b1000_0000_0000_0000_0000_0000_0000_0000; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:blue;">const</span>&nbsp;<span style="color:blue;">uint</span>&nbsp;ratingMask&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;0b0111_1000_0000_0000_0000_0000_0000_0000; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">private</span>&nbsp;<span style="color:blue;">readonly</span>&nbsp;<span style="color:blue;">uint</span>&nbsp;bits; + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:#2b91af;">Song</span>(<span style="color:blue;">int</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>,&nbsp;<span style="color:blue;">bool</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">isVerifiedArtist</span>,&nbsp;<span style="color:blue;">byte</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">rating</span>) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">idBits</span>&nbsp;=&nbsp;(<span style="color:blue;">uint</span>)id&nbsp;&amp;&nbsp;idMask; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">isVerifiedArtistBits</span>&nbsp;=&nbsp;isVerifiedArtist&nbsp;?&nbsp;isVerifiedArtistMask&nbsp;:&nbsp;0u; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">ratingBits</span>&nbsp;=&nbsp;((<span style="color:blue;">uint</span>)rating&nbsp;&lt;&lt;&nbsp;27)&nbsp;&amp;&nbsp;ratingMask; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;bits&nbsp;=&nbsp;idBits&nbsp;|&nbsp;isVerifiedArtistBits&nbsp;|&nbsp;ratingBits; +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">int</span>&nbsp;Id&nbsp;=&gt;&nbsp;(<span style="color:blue;">int</span>)(bits&nbsp;&amp;&nbsp;idMask); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">bool</span>&nbsp;IsVerifiedArtist&nbsp;=&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;(bits&nbsp;&amp;&nbsp;isVerifiedArtistMask)&nbsp;==&nbsp;isVerifiedArtistMask; + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">byte</span>&nbsp;Rating&nbsp;=&gt;&nbsp;(<span style="color:blue;">byte</span>)((bits&nbsp;&amp;&nbsp;ratingMask)&nbsp;&gt;&gt;&nbsp;27); +}</pre> + </p> + <p> + In this representation, I've set aside the lower 27 bits for the ID, enabling IDs to range between 0 and 134,217,727. The top bit is used for <code>IsVerifiedArtist</code>, and the remaining four bits for <code>Rating</code>. + </p> + <p> + This data structure only holds a single <code>uint</code>, and since I made it a <code>struct</code>, I thought it'd have minimal overhead. + </p> + <p> + As you can see in the above chart, that's not the case. When I run the experiment, this representation requires <em>more</em> memory. + </p> + <p> + Just to make sure I wasn't missing anything obvious, I tried making the bitmasked <code>Song</code> a <code>class</code> instead. No visible difference. + </p> + <p> + If you're wondering why the bitmasked data series only go to 500,000, it's because this change made the experiments glacial. It took somewhere between 12 and 24 hours to run the experiment with a <code>size</code> of 500,000. + </p> + <p> + For what it's worth, I don't think the slowdown is directly related to the data representation, but rather to the change I had to make to the <a href="https://fscheck.github.io/FsCheck/">FsCheck</a>-based data generator: + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;songParams&nbsp;=&nbsp;gen&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;maxId&nbsp;=&nbsp;0b0111_1111_1111_1111_1111_1111_1111 +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;songId&nbsp;=&nbsp;Gen.choose&nbsp;(1,&nbsp;maxId) +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;isVerified&nbsp;=&nbsp;ArbMap.generate&nbsp;ArbMap.defaults +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;rating&nbsp;=&nbsp;Gen.choose&nbsp;(0,&nbsp;10)&nbsp;|&gt;&nbsp;Gen.map&nbsp;byte +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;songId,&nbsp;isVerified,&nbsp;rating&nbsp;} + +[&lt;CompiledName&nbsp;<span style="color:#a31515;">&quot;Song&quot;</span>&gt;] +<span style="color:blue;">let</span>&nbsp;song&nbsp;=&nbsp;gen&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;(id,&nbsp;isVerifiedArtist,&nbsp;rating)&nbsp;=&nbsp;songParams +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;Song&nbsp;(id,&nbsp;isVerifiedArtist,&nbsp;rating)&nbsp;}</pre> + </p> + <p> + I can't explain why the bitmasked representation requires more memory, but I'm increasingly having a nagging feeling that I've made a mistake somewhere. If you can spot a mistake, please let me know by leaving a comment. + </p> + <h3 id="90e12771e0214d239cc2bc1c6a540c0a"> + Other data representations <a href="#90e12771e0214d239cc2bc1c6a540c0a">#</a> + </h3> + <p> + I also considered whether it'd make sense to represent the entire data set as a huge matrix. One could, for example, let rows represent users, and columns songs, and let each element represent the number of times a user has listened to a particular song: + </p> + <table> + <tr> + <td>User</td> + <td>Song 1</td> + <td>Song 2</td> + <td>Song 3</td> + <td>...</td> + </tr> + <tr> + <td>123</td> + <td>0</td> + <td>0</td> + <td>4</td> + <td>...</td> + </tr> + <tr> + <td>456</td> + <td>2</td> + <td>0</td> + <td>4</td> + <td>...</td> + </tr> + <tr> + <td colspan="5">...</td> + </tr> + </table> + <p> + Let's say that you may expect some users to listen to a song more than 255 times, but probably not more than 65,535 times. Thus, you could store each play count as a <a href="https://learn.microsoft.com/dotnet/api/system.uint16">ushort</a>. Still, you would need <em>users x songs</em> values, so if you have 100 million songs and 10 million users, that implies 2 PB of memory. That doesn't sound useful. + </p> + <p> + On the other hand, most of those elements are going to be <em>0</em>, so perhaps one could use an <a href="https://en.wikipedia.org/wiki/Adjacency_list">adjacency list</a> instead. That is, however, essentially what an <code>IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;</code> is, so we're now back where we started. + </p> + <h3 id="80eaf0d1dd9848cb875f74b94cb64054"> + Conclusion <a href="#80eaf0d1dd9848cb875f74b94cb64054">#</a> + </h3> + <p> + This article reports on some measurements I've made of memory requirements, assuming that we keep all scrobble data in memory. While I suspect that I've made a mistake, it still seems reasonable to conclude that the <em>song recommendations</em> scenario is on the edge of what's possible with the Impureim Sandwich pattern. + </p> + <p> + That's okay. I'm interested in understanding the limitations of solutions. + </p> + <p> + I do think, however, that it's worth taking note of just how massive amounts of data are required before the Impureim Sandwich pattern becomes untenable. + </p> + <p> + When I describe the pattern, the most common reaction is that it doesn't scale. And, as this article has explored, it's true that it doesn't scale. But it scales much, much better than you think. You may have billions of entities in your system, and they may still fit in a few gigabytes. Don't dismiss the Impureim Sandwich before you've made a real effort to understand the memory implications of it. Your intuition is likely to work against you. + </p> + <p> + I'll round off this part of the article series by showing how the Impureim Sandwich looks in other, more functional languages. + </p> + <p> + <strong>Next:</strong> <a href="/2025/05/19/song-recommendations-as-an-f-impureim-sandwich">Song recommendations as an F# Impureim Sandwich</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/05/12/song-recommendations-proof-of-concept-memory-measurements + + + + Song recommendations as a C# Impureim Sandwich + https://blog.ploeh.dk/2025/05/05/song-recommendations-as-a-c-impureim-sandwich/ + Mon, 05 May 2025 06:23:00 UTC + + + +<div id="post"> + <p> + <em>A refactoring example.</em> + </p> + <p> + This is an article in a <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">larger series about functional programming design alternatives</a>. I'm assuming that you've read the previous articles, but briefly told, I'm considering an example presented by <a href="https://tyrrrz.me/">Oleksii Holub</a> in the article <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">Pure-Impure Segregation Principle</a>. The example gathers song recommendations for a user in <a href="https://twitter.com/Tyrrrz/status/1493369905869213700">a long-running process</a>. + </p> + <p> + In <a href="/2025/04/28/song-recommendations-as-an-impureim-sandwich">the previous article</a> I argued that while the memory requirements for this problem seem so vast that an <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> appears out of the question, it's nonetheless worthwhile to at least try it out. The refactoring isn't that difficult, and it turns out that it does simplify the code. + </p> + <h3 id="1fa2f097e260439fbc0bafb67a35a394"> + Enumeration API <a href="#1fa2f097e260439fbc0bafb67a35a394">#</a> + </h3> + <p> + The data access API is a web service: + </p> + <blockquote> + <p> + "I don't own the database, those are requests to an external API service (think Spotify API) that provides the data." + </p> + <footer><cite><a href="https://twitter.com/Tyrrrz/status/1495465374179119105">Tweet</a></cite>, Oleksii Holub, 2022</footer> + </blockquote> + <p> + In order to read all data, we'll have to assume that there's a way to enumerate all songs and all users. With that assumption, I add the <code>GetAllSongs</code> and <code>GetAllUsers</code> methods to the <code>SongService</code> interface: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">interface</span>&nbsp;<span style="color:#2b91af;">SongService</span> +{ +&nbsp;&nbsp;&nbsp;&nbsp;Task&lt;IEnumerable&lt;Song&gt;&gt;&nbsp;GetAllSongs(); +&nbsp;&nbsp;&nbsp;&nbsp;Task&lt;IEnumerable&lt;User&gt;&gt;&nbsp;GetAllUsers(); +&nbsp;&nbsp;&nbsp;&nbsp;Task&lt;IReadOnlyCollection&lt;User&gt;&gt;&nbsp;GetTopListenersAsync(<span style="color:blue;">int</span>&nbsp;songId); +&nbsp;&nbsp;&nbsp;&nbsp;Task&lt;IReadOnlyCollection&lt;Scrobble&gt;&gt;&nbsp;GetTopScrobblesAsync(<span style="color:blue;">string</span>&nbsp;userName); +}</pre> + </p> + <p> + It is, of course, a crucial assumption, and it's possible that no such API exists. On the other hand, a REST API could expose such functionality as a paged feed. Leafing through potentially hundreds (or thousands) such pages is bound to take some time, so it's good to know that this is a background process. As I briefly mentioned in <a href="/2025/04/28/song-recommendations-as-an-impureim-sandwich">the previous article</a>, we could imagine that we have a dedicated indexing server for this kind purpose. While we may rightly expect the initial data load to take some time (hours, even), once it's in memory, we should be able to reuse it to calculate song recommendations for all users, instead of just one user. + </p> + <p> + In the previous article I estimated that it should be possible to keep all songs in memory with less than a gigabyte. Users, without scrobbles, also take up surprisingly little space, to the degree that a million users fit in a few dozen megabytes. Even if, eventually, we may be concerned about memory, we don't have to be concerned about this part. + </p> + <p> + In any case, the addition of these two new methods doesn't break the existing example code, although I did have to implement the method in the <code>FakeSongService</code> class that I introduced in the article <a href="/2025/04/10/characterising-song-recommendations">Characterising song recommendations</a>: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;Task&lt;IEnumerable&lt;Song&gt;&gt;&nbsp;GetAllSongs() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;Task.FromResult&lt;IEnumerable&lt;Song&gt;&gt;(songs.Values); +} + +<span style="color:blue;">public</span>&nbsp;Task&lt;IEnumerable&lt;User&gt;&gt;&nbsp;GetAllUsers() +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;Task.FromResult(users.Select(kvp&nbsp;=&gt;&nbsp;<span style="color:blue;">new</span>&nbsp;User(kvp.Key,&nbsp;kvp.Value.Values.Sum()))); +}</pre> + </p> + <p> + With those additions, we can load all data as the first layer (<em>phase</em>, really) of the sandwich. + </p> + <h3 id="9e69e4d3be8f4562b0cf9369610f6ebb"> + Front-loading the data <a href="#9e69e4d3be8f4562b0cf9369610f6ebb">#</a> + </h3> + <p> + Loading all the data is the responsibility of this <code>DataCollector</code> module: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:blue;">class</span>&nbsp;<span style="color:#2b91af;">DataCollector</span> +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:blue;">async</span>&nbsp;Task&lt;IReadOnlyDictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;CollectAllTopListeners(<span style="color:blue;">this</span>&nbsp;SongService&nbsp;songService) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;dict&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;Dictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">foreach</span>&nbsp;(var&nbsp;song&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="color:blue;">await</span>&nbsp;songService.GetAllSongs()) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;topListeners&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;songService.GetTopListenersAsync(song.Id); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;dict.Add(song.Id,&nbsp;topListeners); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;} +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;dict; +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;<span style="color:blue;">async</span>&nbsp;Task&lt;IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;CollectAllTopScrobbles(<span style="color:blue;">this</span>&nbsp;SongService&nbsp;songService) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;dict&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;Dictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">foreach</span>&nbsp;(var&nbsp;user&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="color:blue;">await</span>&nbsp;songService.GetAllUsers()) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;topScrobbles&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;songService.GetTopScrobblesAsync(user.UserName); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;dict.Add(user.UserName,&nbsp;topScrobbles); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;} +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;dict; +&nbsp;&nbsp;&nbsp;&nbsp;} +}</pre> + </p> + <p> + These two methods work with any <code>SongService</code> implementation, so while the code base will work with <code>FakeSongService</code>, real 'production code' might as well use an HTTP-based implementation that pages through the implied web API. + </p> + <p> + The dictionaries returned by the methods are likely to be huge. That's a major point of this exercise. Once the change is implemented and <a href="https://en.wikipedia.org/wiki/Characterization_test">Characterisation Tests</a> show that it still works, it makes sense to generate data to get a sense of the memory footprint. + </p> + <h3 id="cad69cb800a146e9ac2556c7deaac655"> + Table-driven methods <a href="#cad69cb800a146e9ac2556c7deaac655">#</a> + </h3> + <p> + Perhaps you wonder why the above <code>CollectAllTopListeners</code> and <code>CollectAllTopScrobbles</code> methods return dictionaries of exactly that shape. + </p> + <p> + <a href="http://amzn.to/1dLYr0r">Code Complete</a> describes a programming technique called <em>table-driven methods</em>. The idea is to replace branching instructions such as <code>if</code>, <code>else</code>, and <code>switch</code> with a lookup table. The overall point, however, is that you can replace function calls with table lookups. + </p> + <p> + Consider the <code>GetTopListenersAsync</code> method. It takes an <code>int</code> as input, and returns a <code>Task&lt;IReadOnlyCollection&lt;User&gt;&gt; + </code> as output. If you ignore the <code>Task</code>, that's an <code>IReadOnlyCollection&lt;User&gt;</code>. In other words, you can exchange an <code>int</code> for an <code>IReadOnlyCollection&lt;User&gt;</code>. + </p> + <p> + If you have an <code>IReadOnlyDictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;</code> you can <em>also</em> exchange an <code>int</code> for an <code>IReadOnlyCollection&lt;User&gt;</code>. These two APIs are functionally equivalent - although, of course, they have very different memory and run-time profiles. + </p> + <p> + The same goes for the <code>GetTopScrobblesAsync</code> method: It takes a <code>string</code> as input and returns an <code>IReadOnlyCollection&lt;Scrobble&gt;</code> as output (if you ignore the <code>Task</code>). An <code>IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;</code> is equivalent. + </p> + <p> + To make it practical, it turns out that we also need a little helper method to deal with the case where the dictionary has no entry for a given key: + </p> + <p> + <pre><span style="color:blue;">internal</span>&nbsp;<span style="color:blue;">static</span>&nbsp;IReadOnlyCollection&lt;T&gt;&nbsp;GetOrEmpty&lt;<span style="color:#2b91af;">T</span>,&nbsp;<span style="color:#2b91af;">TKey</span>&gt;( +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">this</span>&nbsp;IReadOnlyDictionary&lt;TKey,&nbsp;IReadOnlyCollection&lt;T&gt;&gt;&nbsp;dict, +&nbsp;&nbsp;&nbsp;&nbsp;TKey&nbsp;key) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">if</span>&nbsp;(dict.TryGetValue(key,&nbsp;<span style="color:blue;">out</span>&nbsp;var&nbsp;result)) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;result; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;Array.Empty&lt;T&gt;(); +}</pre> + </p> + <p> + If there's no entry for a key, this function instead returns an empty array. + </p> + <p> + That should make it as easy as possible to replace calls to <code>GetTopListenersAsync</code> and <code>GetTopScrobblesAsync</code> with dictionary lookups. + </p> + <h3 id="43b6cc924db64320a41f0a4093c65e29"> + Adding method parameters <a href="#43b6cc924db64320a41f0a4093c65e29">#</a> + </h3> + <p> + When refactoring, it's a good idea to proceed in small, controlled steps. You can see each of my <a href="https://www.industriallogic.com/blog/whats-this-about-micro-commits/">micro-commits</a> in the Git repository's <em>refactor-to-function</em> branch. Here, I'll give an overview. + </p> + <p> + First, I added two dictionaries as parameters to the <code>GetRecommendationsAsync</code> method. You may recall that the method used to look like this: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;Task&lt;IReadOnlyList&lt;Song&gt;&gt;&nbsp;GetRecommendationsAsync(<span style="color:blue;">string</span>&nbsp;userName)</pre> + </p> + <p> + After I added the two dictionaries, it looks like this: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">async</span>&nbsp;Task&lt;IReadOnlyList&lt;Song&gt;&gt;&nbsp;GetRecommendationsAsync( +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">string</span>&nbsp;userName, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;&nbsp;topScrobbles, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;&nbsp;topListeners)</pre> + </p> + <p> + At this point, the <code>GetRecommendationsAsync</code> method uses neither the <code>topScrobbles</code> nor the <code>topListeners</code> parameter. Still, I consider this a <a href="https://stackoverflow.blog/2022/12/19/use-git-tactically/">distinct checkpoint that I commit to Git</a>. As I've outlined in my book <a href="/2021/06/14/new-book-code-that-fits-in-your-head">Code That Fits in Your Head</a>, it's safest to either refactor production code while keeping test code untouched, or refactor test code without editing the production code. An API change like the current is an example of a situation where that separation is impossible. This is the reason I want to keep it small and isolated. While the change does touch both production code and test code, I'm not editing the behaviour of the System Under Test. + </p> + <p> + Tests now look like this: + </p> + <p> + <pre>[&lt;Property&gt;] +<span style="color:blue;">let</span>&nbsp;``No&nbsp;data``&nbsp;()&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;Gen.userName&nbsp;|&gt;&nbsp;Arb.fromGen&nbsp;|&gt;&nbsp;Prop.forAll&nbsp;&lt;|&nbsp;<span style="color:blue;">fun</span>&nbsp;userName&nbsp;<span style="color:blue;">-&gt;</span> +&nbsp;&nbsp;&nbsp;&nbsp;task&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;srvc&nbsp;=&nbsp;FakeSongService&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;sut&nbsp;=&nbsp;RecommendationsProvider&nbsp;srvc + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;topScrobbles&nbsp;=&nbsp;DataCollector.CollectAllTopScrobbles&nbsp;srvc +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;topListeners&nbsp;=&nbsp;DataCollector.CollectAllTopListeners&nbsp;srvc +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;actual&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sut.GetRecommendationsAsync&nbsp;(userName,&nbsp;topScrobbles,&nbsp;topListeners) + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Assert.Empty&nbsp;actual&nbsp;}&nbsp;:&gt;&nbsp;Task</pre> + </p> + <p> + The test now uses the <code>DataCollector</code> to front-load data into dictionaries that it then passes to <code>GetRecommendationsAsync</code>. Keep in mind that <code>GetRecommendationsAsync</code> doesn't yet use that data, but it's available to it once I make a change to that effect. + </p> + <p> + You may wish to compare this version of the <code>No data</code> test with the previous version shown in the article <a href="/2025/04/10/characterising-song-recommendations">Characterising song recommendations</a>. + </p> + <h3 id="02c3e9706d044528ad1d5acec6865385"> + Refactoring to function <a href="#02c3e9706d044528ad1d5acec6865385">#</a> + </h3> + <p> + The code is now ready for refactoring <a href="/2017/01/27/from-dependency-injection-to-dependency-rejection">from dependency injection to dependency rejection</a>. It's even possible to do it one method call at a time, because the data in the <code>FakeSongService</code> is the same as the data in the two dictionaries. It's just two different representations of the same data. + </p> + <p> + Since, as described above, the dictionaries are equivalent to the <code>SongService</code> queries, each is easily replaced with the other. The first impure action in <code>GetRecommendationsAsync</code>, for example, is this one: + </p> + <p> + <pre><span style="color:blue;">var</span>&nbsp;scrobbles&nbsp;=&nbsp;<span style="color:blue;">await</span>&nbsp;_songService.GetTopScrobblesAsync(userName);</pre> + </p> + <p> + The equivalent dictionary lookup enables us to change that line of code to this: + </p> + <p> + <pre><span style="color:blue;">var</span>&nbsp;scrobbles&nbsp;=&nbsp;topScrobbles.GetOrEmpty(userName);</pre> + </p> + <p> + Notice that the dictionary lookup is a <a href="https://en.wikipedia.org/wiki/Pure_function">pure function</a> that the method need not <code>await</code>. + </p> + <p> + Even though the rest of <code>GetRecommendationsAsync</code> still queries the injected <code>SongService</code>, all tests pass, and I can commit this small, isolated change to Git. + </p> + <p> + Proceeding in a similar fashion enables us to eliminate the <code>SongService</code> queries one by one. There are only three method calls, so this can be done in three controlled steps. Once the last impure query has been replaced, the C# compiler complains about the <code>async</code> keyword in the declaration of the <code>GetRecommendationsAsync</code> method. + </p> + <p> + Not only is the <code>async</code> keyword no longer required, the method is no longer asynchronous. There's no reason to return a <code>Task</code>, and the <code>Async</code> method name suffix is also misleading. + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;IReadOnlyList&lt;Song&gt;&nbsp;GetRecommendations( +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">string</span>&nbsp;userName, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;&nbsp;topScrobbles, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;&nbsp;topListeners)</pre> + </p> + <p> + The <code>GetRecommendations</code> method no longer uses the injected <code>SongService</code>, and since it's is the only method of the <code>RecommendationsProvider</code> class, we can now (r)eject the dependency. + </p> + <p> + This furthermore means that the class no longer has any class fields; we might as well make it (and the <code>GetRecommendations</code> function) <code>static</code>. Here's the final function in its entirety: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">static</span>&nbsp;IReadOnlyList&lt;Song&gt;&nbsp;GetRecommendations( +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">string</span>&nbsp;userName, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">string</span>,&nbsp;IReadOnlyCollection&lt;Scrobble&gt;&gt;&nbsp;topScrobbles, +&nbsp;&nbsp;&nbsp;&nbsp;IReadOnlyDictionary&lt;<span style="color:blue;">int</span>,&nbsp;IReadOnlyCollection&lt;User&gt;&gt;&nbsp;topListeners) +{ +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;scrobbles&nbsp;=&nbsp;topScrobbles.GetOrEmpty(userName); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;scrobblesSnapshot&nbsp;=&nbsp;scrobbles +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.OrderByDescending(s&nbsp;=&gt;&nbsp;s.ScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.Take(100) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.ToArray(); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;recommendationCandidates&nbsp;=&nbsp;<span style="color:blue;">new</span>&nbsp;List&lt;Song&gt;(); +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">foreach</span>&nbsp;(var&nbsp;scrobble&nbsp;<span style="color:blue;">in</span>&nbsp;scrobblesSnapshot) +&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;otherListeners&nbsp;=&nbsp;topListeners.GetOrEmpty(scrobble.Song.Id); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;otherListenersSnapshot&nbsp;=&nbsp;otherListeners +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.Where(u&nbsp;=&gt;&nbsp;u.TotalScrobbleCount&nbsp;&gt;=&nbsp;10_000) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.OrderByDescending(u&nbsp;=&gt;&nbsp;u.TotalScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.Take(20) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.ToArray(); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">foreach</span>&nbsp;(var&nbsp;otherListener&nbsp;<span style="color:blue;">in</span>&nbsp;otherListenersSnapshot) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;otherScrobbles&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;topScrobbles.GetOrEmpty(otherListener.UserName); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;otherScrobblesSnapshot&nbsp;=&nbsp;otherScrobbles +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.Where(s&nbsp;=&gt;&nbsp;s.Song.IsVerifiedArtist) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.OrderByDescending(s&nbsp;=&gt;&nbsp;s.Song.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.Take(10) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.ToArray(); + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;recommendationCandidates.AddRange( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;otherScrobblesSnapshot.Select(s&nbsp;=&gt;&nbsp;s.Song) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;); +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;} +&nbsp;&nbsp;&nbsp;&nbsp;} + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">var</span>&nbsp;recommendations&nbsp;=&nbsp;recommendationCandidates +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.OrderByDescending(s&nbsp;=&gt;&nbsp;s.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.Take(200) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;.ToArray(); + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;recommendations; +}</pre> + </p> + <p> + The overall structure is similar to <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">the original version</a>. Now that the code is simpler (because there's no longer any asynchrony) you could keep refactoring. With this C# code example, I'm going to stop here, but when I port it to <a href="https://fsharp.org/">F#</a> I'm going to refactor more aggressively. + </p> + <h3 id="b421654127ca4580995e008d4c01e3ab"> + Sandwich <a href="#b421654127ca4580995e008d4c01e3ab">#</a> + </h3> + <p> + One point of the whole exercise is to demonstrate how to refactor to an Impureim Sandwich. The <code>GetRecommendations</code> method shown above constitutes the pure filling of the sandwich, but what does the entire sandwich look like? + </p> + <p> + In this code base, the sandwiches only exist as unit tests, the simplest of which is still the <code>No data</code> test: + </p> + <p> + <pre>[&lt;Property&gt;] +<span style="color:blue;">let</span>&nbsp;``No&nbsp;data``&nbsp;()&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;Gen.userName&nbsp;|&gt;&nbsp;Arb.fromGen&nbsp;|&gt;&nbsp;Prop.forAll&nbsp;&lt;|&nbsp;<span style="color:blue;">fun</span>&nbsp;user&nbsp;<span style="color:blue;">-&gt;</span> +&nbsp;&nbsp;&nbsp;&nbsp;task&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;srvc&nbsp;=&nbsp;FakeSongService&nbsp;() + +<span style="background-color: lightsalmon;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;topScrobbles&nbsp;=&nbsp;DataCollector.CollectAllTopScrobbles&nbsp;srvc +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;topListeners&nbsp;=&nbsp;DataCollector.CollectAllTopListeners&nbsp;srvc</span> +<span style="background-color: palegreen;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;actual&nbsp;=&nbsp;RecommendationsProvider.GetRecommendations&nbsp;(user,&nbsp;topScrobbles,&nbsp;topListeners)</span> + +<span style="background-color: lightsalmon;">&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Assert.Empty&nbsp;actual</span>&nbsp;}&nbsp;:&gt;&nbsp;Task</pre> + </p> + <p> + In the above code snippet, I've coloured in the relevant part of the test. I admit that it's a stretch to colour the last line red, since <code>Assert.Empty</code> is, at least, deterministic. One could argue that since it throws an exception on failure, it's not strictly free of side effects, but that's really a weak argument. It would be easy to refactor <a href="/2022/11/07/applicative-assertions">assertions to pure functions</a>. + </p> + <p> + Instead, you may consider the bottom layer of the sandwich as a placeholder where something impure might happen. The background service that updates the song recommendations may, for example, save the result as a (CQRS-style) materialised view. + </p> + <p> + The above test snippet, then, is more of a sketch of how the Impureim Sandwich may look: First, front-load data using the <code>DataCollector</code> methods; second, call <code>GetRecommendations</code>; third, do something with the result. + </p> + <h3 id="26a74728792d494e814b8d86f2ad8531"> + Conclusion <a href="#26a74728792d494e814b8d86f2ad8531">#</a> + </h3> + <p> + The changes demonstrated in this article serve two purposes. One is to show how to refactor an impure action to a pure function, pursuing the notion of an Impureim Sandwich. The second is to evaluate a proof-of-concept: If we do, indeed, front-load all of the data, is it realistic that all <a href="https://yourdatafitsinram.net/">data fits in RAM</a>? + </p> + <p> + We have yet to address that question, but since the present article is already substantial, I'll address that in a separate article. + </p> + <strong>Next:</strong> <a href="/2025/05/12/song-recommendations-proof-of-concept-memory-measurements">Song recommendations proof-of-concept memory measurements</a>. +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/05/05/song-recommendations-as-a-c-impureim-sandwich + + + + Song recommendations as an Impureim Sandwich + https://blog.ploeh.dk/2025/04/28/song-recommendations-as-an-impureim-sandwich/ + Mon, 28 Apr 2025 07:16:00 UTC + + + +<div id="post"> + <p> + <em>Does your data fit in RAM?</em> + </p> + <p> + This article is part of <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">a series on functional programming design alternatives</a>. In a <a href="/2025/04/10/characterising-song-recommendations">previous article</a> you saw how to add enough <a href="https://en.wikipedia.org/wiki/Characterization_test">Characterisation Tests</a> to capture the intended behaviour of the example song recommendations system originally presented by <a href="https://tyrrrz.me/">Oleksii Holub</a> in the article <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">Pure-Impure Segregation Principle</a>. + </p> + <h3 id="572693e6fb56455a958ca0d62aa13319"> + Problem statement <a href="#572693e6fb56455a958ca0d62aa13319">#</a> + </h3> + <p> + After showing how one problem can be refactored to <a href="https://en.wikipedia.org/wiki/Pure_function">pure functions</a>, Oleksii Holub writes: + </p> + <blockquote> + <p> + "Although very useful, the type of "lossless" refactoring shown earlier only works if the data required by the function can be easily encapsulated within its input parameters. Unfortunately, this is not always the case. + </p> + <p> + "Often a function may need to dynamically resolve data from an external API or a database, with no way of knowing about it beforehand. This typically results in an implementation where pure and impure concerns are interleaved with each other, creating a tightly coupled cohesive structure." + </p> + <footer><cite>Oleksii Holub, <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">Pure-Impure Segregation Principle</a></cite></footer> + </blockquote> + <p> + The article then proceeds to present the <em>song recommendations</em> example. It's a single C# method that queries a data store or third-party service to recommend songs. I'm imagining that it queries a third-party web service that contains usages data for a system like <a href="https://en.wikipedia.org/wiki/Spotify">Spotify</a>. + </p> + <blockquote> + <p> + "The above algorithm works by retrieving the user's most listened songs, finding other people who have also listened to the same titles, and then extracting their top songs as well. Those songs are then aggregated into a list of recommendations and returned to the caller. + </p> + <p> + "It's quite clear that this function would benefit greatly from being pure, seeing how much business logic is encapsulated within it. Unfortunately, the technique we relied upon earlier won't work here. + </p> + <p> + "In order to fully isolate <code>GetRecommendationsAsync(...)</code> from its impure dependencies, we would have to somehow supply the function with an entire list of songs, users, and their scrobbles upfront. If we assume that we're dealing with data on millions of users, it's obvious that this would be completely impractical and likely even impossible." + </p> + <footer><cite>Oleksii Holub, <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">Pure-Impure Segregation Principle</a></cite></footer> + </blockquote> + <p> + It does, indeed, sound impractical. + </p> + <h3 id="ca5fdd1711554b3eb1ae7d516c003959"> + Data sizes <a href="#ca5fdd1711554b3eb1ae7d516c003959">#</a> + </h3> + <p> + Can you, however, trust your intuition? Research suggests that the human brain is ill-equipped to think about randomness and probabilities, and I've observed something similar when it comes to data sizes. + </p> + <p> + <img src="/content/binary/dr-evil-one-million.jpg" alt="Dr. Evil: One million."> + </p> + <p> + In the real world, a million of anything countable is an almost incomprehensible amount, so it's no wonder if our intuition fails us. <em>A million records</em> sounds like a lot, but if it's only a few integers, is it really that bad? + </p> + <p> + Many systems use 32-bit integers for various IDs. A million IDs, then, is 32 million bits, or approximately 4 MB. As I'm writing this, the smallest Azure instance (<em>Free F1</em>) has 1 GB of memory, and while the OS takes a big bite out of that, 4 MB is nothing. + </p> + <p> + The <em>song recommendations</em> problem implies larger memory pressure. It may not fit on every machine, but it's worth considering if, after all, it doesn't fit in RAM. + </p> + <h3 id="0579cae8c684409f8a43eaf2baa1d5b0"> + My real-life experience with developing streaming services <a href="#0579cae8c684409f8a43eaf2baa1d5b0">#</a> + </h3> + <p> + It just so happens that I have professional experience developing REST APIs for a white-label audio streaming service. Back in the early 2010s I helped design and implement the company's online music catalogue, user system, and a few other services. The catalogue is particularly interesting in this regard, since it only changed nightly, and we were planning on relying on HTTP for caching. + </p> + <p> + I vividly recall a meeting we had with the IT operations specialist responsible for the new catalogue service. We explained that we'd set HTTP cache timeouts to 6 hours, and asked if he'd be able to set up a <a href="https://en.wikipedia.org/wiki/Reverse_proxy">reverse proxy</a> so that we didn't have to implement caching in our code base. + </p> + <p> + He asked how much cache space we needed. + </p> + <p> + We knew the size of a typical HTTP response, and the number of tracks, artists, and albums in the system, so after a back-of-the-envelope calculation, we told him: 18 GB. + </p> + <p> + He just shrugged and said <em>"OK"</em>. + </p> + <p> + In 2012 I though that 18 GB was a fair amount of data (I actually still think that). Even so, the operations team had plenty of servers with that capacity. + </p> + <p> + Later, I did more work for that company, but most of it is less relevant to the <em>song recommendations</em> example. What does turn out to be relevant to the topic is something I learned the first few days of my engagement. + </p> + <p> + Early on, before I was involved, the company needed a recommendation system, but hadn't been able to find any off-the-shelf component. This was in the early 2000s and before <a href="https://en.wikipedia.org/wiki/Apache_Solr">Solr</a>, but after <a href="https://en.wikipedia.org/wiki/Apache_Lucene">Lucene</a>. I'm not aware of all the forces that affected my then future client, but in the end, they decided to write their own search and recommendations engine. + </p> + <p> + Essentially, during the night a beefy server would read all relevant data from the database, crunch it, create data structures, and keep all data in memory. Like the reverse proxy, it required a server with more RAM than a normal <a href="https://en.wikipedia.org/wiki/Pizza_box_form_factor">pizza box</a>, but not prohibitively so. + </p> + <h3 id="8fa70bcec92b4b8681af7ca819e82a22"> + Costs <a href="#8fa70bcec92b4b8681af7ca819e82a22">#</a> + </h3> + <p> + Consider the cost of hardware, compared to developer time. A few specialised servers may set your organisation back a few thousand of dollars/pounds/euros. That's an amount you can easily burn through in salary if the code is too complicated, or has too many bugs. + </p> + <p> + You may argue that if you already have programmers on staff, they don't cost extra, but a too-complicated code base is still going to slow them down. Thus, the wrong software design could incur an <a href="https://en.wikipedia.org/wiki/Opportunity_cost">opportunity cost</a> greater than the cost of a server. + </p> + <p> + One of many reasons I'm interested in functional programming (FP) is its potential to make code bases simpler. The <a href="/2020/03/02/impureim-sandwich">Impureim Sandwich</a> is a wonderfully simple design, so it's worth pursuing; not only for some FP ideal, but because of its simplifying potential. + </p> + <p> + Intuition may tell us that the <em>song recommendations</em> scenario is prohibitively big, and therefore, an Impureim Sandwich is out of the question. As this overall article series explores, it's not the only alternative, but given its advantages, its worth giving it a second chance. + </p> + <h3 id="ac5d135ca06a472e88971ad6a8960aac"> + Analysis <a href="#ac5d135ca06a472e88971ad6a8960aac">#</a> + </h3> + <p> + The <code>GetRecommendationsAsync</code> method from <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle#interleaved-impurities">the example</a> makes a lot of external calls, with its nested loops. The method uses the first call to <code>GetTopScrobblesAsync</code> to produce the <code>scrobblesSnapshot</code> variable, which is capped at 100 objects. If we assume that this method call returns at least 100 objects, the outer <code>foreach</code> loop will make 100 calls to <code>GetTopListenersAsync</code>. + </p> + <p> + If we again assume that each of these return enough data, the inner <code>foreach</code> loop will make 20 calls to <code>GetTopScrobblesAsync</code>, for each object in the outer loop. That's 2,000 external calls, plus the 100 calls in the outer loop, plus the initial call to <code>GetTopScrobblesAsync</code>, for a total of 2,101. + </p> + <p> + When I first saw the example, I didn't know much about the overall context. I didn't know if these impure actions were database queries or web service calls, so I asked Oleksii Holub. + </p> + <p> + It turns out that it's all web service calls, and as I interpret the response, <code>GetRecommendationsAsync</code> is being invoked from a background maintenance process. + </p> + <blockquote> + <p> + "It takes around 10 min in total while maintaining it." + </p> + <footer><cite><a href="https://twitter.com/Tyrrrz/status/1493369905869213700">Tweet</a>, Oleksii Holub, 2022</cite></footer> + </blockquote> + <p> + That's good to know, because if we're going to consider an Impureim Sandwich, it implies reading gigabytes of data in the first phase. That's going to take some time, but if this is a background process, we <em>do</em> have time. + </p> + <h3 id="2160fbaf60434edd9554ff585de5df2b"> + Memory estimates <a href="#2160fbaf60434edd9554ff585de5df2b">#</a> + </h3> + <p> + One thing is to load an entire song catalogue into memory. That's what required 18 GB in 2012. Another thing is to load all scrobbles; i.e. statistics about plays. Fortunately, in order to produce song recommendations, we only need IDs. Consider again the data structures from the <a href="/2025/04/10/characterising-song-recommendations">previous article</a>: + </p> + <p> + <pre><span style="color:blue;">public</span>&nbsp;<span style="color:blue;">sealed</span>&nbsp;<span style="color:blue;">record</span>&nbsp;<span style="color:#2b91af;">Song</span>(<span style="color:blue;">int</span>&nbsp;Id,&nbsp;<span style="color:blue;">bool</span>&nbsp;IsVerifiedArtist,&nbsp;<span style="color:blue;">byte</span>&nbsp;Rating); + +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">sealed</span>&nbsp;<span style="color:blue;">record</span>&nbsp;<span style="color:#2b91af;">Scrobble</span>(Song&nbsp;Song,&nbsp;<span style="color:blue;">int</span>&nbsp;ScrobbleCount); + +<span style="color:blue;">public</span>&nbsp;<span style="color:blue;">sealed</span>&nbsp;<span style="color:blue;">record</span>&nbsp;<span style="color:#2b91af;">User</span>(<span style="color:blue;">string</span>&nbsp;UserName,&nbsp;<span style="color:blue;">int</span>&nbsp;TotalScrobbleCount);</pre> + </p> + <p> + Apart from the <code>UserName</code> all values are small predictable values: <code>int</code>, <code>byte</code>, and <code>bool</code>, and while a <code>string</code> may be arbitrarily long, we can make a guess at the average size of a user name. In the <a href="/2025/04/10/characterising-song-recommendations">previous article</a>, I assumed that the user name would be an alphanumeric string between one and twenty characters. + </p> + <p> + How many songs might a system contain? Some numbers thrown around for a system like Spotify suggest a number on the order of 100 million. With an <code>int</code>, a <code>bool</code>, and a <code>byte</code>, we can estimate that a song requires 6 bytes, plus some overhead. Let's guess 8 bytes. A 100 million songs would then require 800 million bytes, or around 800 MB. That eliminates the smallest cloud instances, but is in itself easily within reach for all modern computers. Your phone has more memory than that. + </p> + <p> + How about scrobbles? While I don't use Spotify, <a href="https://www.last.fm/user/ploeh">I do scrobble plays to Last.fm</a>. At the moment I have around 114,000 scrobbles, and while I don't listen to music as much as I used to when I was younger, I have, on the other hand, been at it for a long time: Since 2007. If we assume that each user has 200,000 scrobbles, and a scrobble requires 8 bytes, that's 1,600,000 bytes, or 1.6 MB. Practically nothing. + </p> + <p> + The size of a <code>User</code> object depends on how long the user name is, but will probably, on average, be less than 32 bytes. Compared to the user's scrobbles, we can ignore the memory pressure of the user object itself. + </p> + <p> + As the number of users grow, it will dominate the memory requirements for the catalogue. How many users should we assume? + </p> + <p> + A million is probably too few, but for a frame of reference, that would require 1,6 TB. This is where it starts to sound unfeasible to keep all data in RAM. Even though servers with that much RAM exist, they're so expensive (still) that the above cost consideration no longer applies. + </p> + <p> + Still, there are some naive assumptions above. Instead of storing each scrobble in a separate <code>Scrobble</code> object, you could store repeated plays as a single object with the appropriate <code>ScrobbleCount</code> value. If you've listened to the same song 50 times, it doesn't require 400 bytes of storage, but only 8 bytes. That is, after all, orders of magnitude less. + </p> + <p> + In the end, back-of-the-envelope calculations are fine, but measurements are better. It might be worthwhile to develop a proof of concept and measure how much memory it requires. + </p> + <p> + In three articles, I'll explore how a <em>song recommendations</em> Impureim Sandwich looks in various constellations: + </p> + <ul> + <li><a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">Song recommendations as a C# Impureim Sandwich</a></li> + <li>Song recommendations as an F# Impureim Sandwich</li> + <li>Song recommendations as a Haskell Impureim Sandwich</li> + </ul> + <p> + In the end, it may turn out that for this particular system, an Impureim Sandwich truly is unfeasible. Keep in mind, though, that the purpose of this article series is to demonstrate alternative designs. The <em>song recommendations</em> problem is just a placeholder example. Perhaps you have another system where, intuitively, an Impureim Sandwich sounds impossible, but once you run the numbers, it might actually be not only possible, but desirable. + </p> + <h3 id="d38416a3535743159ba26abb4a78948b"> + Conclusion <a href="#d38416a3535743159ba26abb4a78948b">#</a> + </h3> + <p> + Modern computers have so vast storage capacities that intuition often fails us. We may think that billions of data points sounds like something that can't possibly fit in RAM. When you run the numbers, however, it may turn out that the required data fits on a normal server. + </p> + <p> + If so, an Impureim Sandwich may still be an option. Load data into memory, pass it as argument to a pure function, and handle the return value. + </p> + <p> + The <em>song recommendations</em> scenario is interesting because an Impureim Sandwich seems to be pushing the envelope. It probably <em>is</em> impractical, but still worth a proof of concept. On the other hand, if it's impractical, it's worthwhile to also explore alternatives. Later articles will do that, but first, if you're interested, the next articles look at the proofs of concept in three languages. + </p> + <p> + <strong>Next:</strong> <a href="/2025/05/05/song-recommendations-as-a-c-impureim-sandwich">Song recommendations as a C# Impureim Sandwich</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/04/28/song-recommendations-as-an-impureim-sandwich + + + + Porting song recommendations to Haskell + https://blog.ploeh.dk/2025/04/21/porting-song-recommendations-to-haskell/ + Mon, 21 Apr 2025 10:19:00 UTC + + + +<div id="post"> + <p> + <em>An F# code base translated to Haskell.</em> + </p> + <p> + This article is part of a <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">larger article series</a> that examines variations of how to take on a non-trivial problem using <a href="/2018/11/19/functional-architecture-a-definition">functional architecture</a>. In a <a href="/2025/04/10/characterising-song-recommendations">previous article</a> we established a baseline C# code base. Future articles are going to use that C# code base as a starting point for refactored code. On the other hand, I also want to demonstrate what such solutions may look like in languages like <a href="https://fsharp.org/">F#</a> or <a href="https://www.haskell.org/">Haskell</a>. In this article, you'll see how to port the baseline to Haskell. To be honest, I first <a href="/2025/04/14/porting-song-recommendations-to-f">ported the C# code to F#</a>, and then used the F# code as a guide to implement equivalent Haskell code. + </p> + <p> + If you're following along in the Git repositories, this is a repository separate from the .NET repositories. The code shown here is from its <em>master</em> branch. + </p> + <p> + If you don't care about Haskell, you can always go back to the table of contents in <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">the 'root' article</a> and proceed to the next topic that interests you. + </p> + <h3 id="2c7511eb050b4d399f7e7fb154c5d990"> + Data structures <a href="#2c7511eb050b4d399f7e7fb154c5d990">#</a> + </h3> + <p> + When working with statically typed functional languages like Haskell, it often makes most sense to start by declaring data structures. + </p> + <p> + <pre><span style="color:blue;">data</span>&nbsp;User&nbsp;=&nbsp;User +&nbsp;&nbsp;{&nbsp;userName&nbsp;::&nbsp;String +&nbsp;&nbsp;,&nbsp;userScrobbleCount&nbsp;::&nbsp;Int&nbsp;} +&nbsp;&nbsp;<span style="color:blue;">deriving</span>&nbsp;(<span style="color:#2b91af;">Show</span>,&nbsp;<span style="color:#2b91af;">Eq</span>)</pre> + </p> + <p> + This is much like an F# or C# record declaration, and this one echoes the corresponding types in F# and C#. The most significant difference is that here, a user's total count of scrobbles is called <code>userScrobbleCount</code> rather than <code>TotalScrobbleCount</code>. The motivation behind that variation is that Haskell data 'getters' are actually top-level functions, so it's usually a good idea to prefix them with the name of the data structure they work on. Since the data structure is called <code>User</code>, both 'getter' functions get the <code>user</code> prefix. + </p> + <p> + I found <code>userTotalScrobbleCount</code> a bit too verbose to my tastes, so I dropped the <code>Total</code> part. Whether or not that's appropriate remains to be seen. Naming in programming is always hard, and there's a risk that you don't get it right the first time around. Unless you're publishing a reusable library, however, the option to rename it later remains. + </p> + <p> + The other two data structures are quite similar: + </p> + <p> + <pre><span style="color:blue;">data</span>&nbsp;Song&nbsp;=&nbsp;Song +&nbsp;&nbsp;{&nbsp;songId&nbsp;::&nbsp;Int +&nbsp;&nbsp;,&nbsp;songHasVerifiedArtist&nbsp;::&nbsp;Bool +&nbsp;&nbsp;,&nbsp;songRating&nbsp;::&nbsp;Word8&nbsp;} +&nbsp;&nbsp;<span style="color:blue;">deriving</span>&nbsp;(<span style="color:#2b91af;">Show</span>,&nbsp;<span style="color:#2b91af;">Eq</span>) + +<span style="color:blue;">data</span>&nbsp;Scrobble&nbsp;=&nbsp;Scrobble +&nbsp;&nbsp;{&nbsp;scrobbledSong&nbsp;::&nbsp;Song +&nbsp;&nbsp;,&nbsp;scrobbleCount&nbsp;::&nbsp;Int&nbsp;} +&nbsp;&nbsp;<span style="color:blue;">deriving</span>&nbsp;(<span style="color:#2b91af;">Show</span>,&nbsp;<span style="color:#2b91af;">Eq</span>)</pre> + </p> + <p> + I thought that <code>scrobbledSong</code> was more descriptive than <code>scrobbleSong</code>, so I allowed myself that little deviation from the <a href="/2015/08/03/idiomatic-or-idiosyncratic">idiomatic</a> naming convention. It didn't cause any problems, but I'm still not sure if that was a good decision. + </p> + <p> + How does one translate a C# interface to Haskell? Although type classes aren't quite the same as C# or Java interfaces, this language feature is close enough that I can use it in that role. I don't consider such a type class idiomatic in Haskell, but as an counterpart to the C# interface, it works well enough. + </p> + <p> + <pre><span style="color:blue;">class</span>&nbsp;<span style="color:#2b91af;">SongService</span>&nbsp;a&nbsp;<span style="color:blue;">where</span> +&nbsp;&nbsp;<span style="color:#2b91af;">getTopListeners</span>&nbsp;<span style="color:blue;">::</span>&nbsp;a&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Int</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">User</span>] +&nbsp;&nbsp;<span style="color:#2b91af;">getTopScrobbles</span>&nbsp;<span style="color:blue;">::</span>&nbsp;a&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">String</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">IO</span>&nbsp;[<span style="color:blue;">Scrobble</span>]</pre> + </p> + <p> + Any instance of the <code>SongService</code> class supports queries for top listeners of a particular song, as well as for top scrobbles for a user. + </p> + <p> + To reiterate, I don't intend to keep this type class around if I can help it, but for didactic reasons, it'll remain in some of the future refactorings, so that you can contrast and compare the Haskell code to its C# and F# peers. + </p> + <h3 id="83350f6a8a484249b466fcb72978c64d"> + Test Double <a href="#83350f6a8a484249b466fcb72978c64d">#</a> + </h3> + <p> + To support tests, I needed a <a href="https://martinfowler.com/bliki/TestDouble.html">Test Double</a>, so I defined the following <a href="http://xunitpatterns.com/Fake%20Object.html">Fake</a> service, which is nothing but a deterministic in-memory instance. The type itself is just a wrapper of two maps. + </p> + <p> + <pre><span style="color:blue;">data</span>&nbsp;FakeSongService&nbsp;=&nbsp;FakeSongService +&nbsp;&nbsp;{&nbsp;fakeSongs&nbsp;::&nbsp;Map&nbsp;Int&nbsp;Song +&nbsp;&nbsp;,&nbsp;fakeUsers&nbsp;::&nbsp;Map&nbsp;String&nbsp;(Map&nbsp;Int&nbsp;Int)&nbsp;} +&nbsp;&nbsp;<span style="color:blue;">deriving</span>&nbsp;(<span style="color:#2b91af;">Show</span>,&nbsp;<span style="color:#2b91af;">Eq</span>)</pre> + </p> + <p> + Like the equivalent C# class, <code>fakeSongs</code> is a map from song ID to <code>Song</code>, while <code>fakeUsers</code> is a bit more complex. It's a map keyed on user name, but the value is another map. The keys of that inner map are song IDs, while the values are the number of times each song was scrobbled by that user. + </p> + <p> + The <code>FakeSongService</code> data structure is a <code>SongService</code> instance by explicit implementation: + </p> + <p> + <pre><span style="color:blue;">instance</span>&nbsp;<span style="color:blue;">SongService</span>&nbsp;<span style="color:blue;">FakeSongService</span>&nbsp;<span style="color:blue;">where</span> +&nbsp;&nbsp;getTopListeners&nbsp;srvc&nbsp;sid&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">uncurry</span>&nbsp;User&nbsp;&lt;$&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Map.toList&nbsp;(<span style="color:blue;">sum</span>&nbsp;&lt;$&gt;&nbsp;Map.<span style="color:blue;">filter</span>&nbsp;(Map.member&nbsp;sid)&nbsp;(fakeUsers&nbsp;srvc)) +&nbsp;&nbsp;getTopScrobbles&nbsp;srvc&nbsp;userName&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">fmap</span>&nbsp;(\(sid,&nbsp;c)&nbsp;-&gt;&nbsp;Scrobble&nbsp;(fakeSongs&nbsp;srvc&nbsp;!&nbsp;sid)&nbsp;c)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Map.toList&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;Map.findWithDefault&nbsp;Map.empty&nbsp;userName&nbsp;(fakeUsers&nbsp;srvc)</pre> + </p> + <p> + In order to find all the top listeners of a song, it finds all the <code>fakeUsers</code> who have the song ID (<code>sid</code>) in their inner map, sum all of those users' scrobble counts together and creates <code>User</code> values from that data. + </p> + <p> + To find the top scrobbles of a user, the instance finds the user in the <code>fakeUsers</code> map, looks each of that user's scrobbled song up in <code>fakeSongs</code>, and creates <code>Scrobble</code> values from that information. + </p> + <p> + Finally, test code needs a way to add data to a <code>FakeSongService</code> value, which this <a href="http://xunitpatterns.com/Test%20Utility%20Method.html">test-specific helper function</a> accomplishes: + </p> + <p> + <pre>scrobble&nbsp;userName&nbsp;s&nbsp;c&nbsp;(FakeSongService&nbsp;ss&nbsp;us)&nbsp;= +&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;sid&nbsp;=&nbsp;songId&nbsp;s +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;ss&#39;&nbsp;=&nbsp;Map.insertWith&nbsp;(\_&nbsp;_&nbsp;-&gt;&nbsp;s)&nbsp;sid&nbsp;s&nbsp;ss +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;us&#39;&nbsp;=&nbsp;Map.insertWith&nbsp;(Map.unionWith&nbsp;<span style="color:#2b91af;">(+)</span>)&nbsp;userName&nbsp;(Map.singleton&nbsp;sid&nbsp;c)&nbsp;us +&nbsp;&nbsp;<span style="color:blue;">in</span>&nbsp;FakeSongService&nbsp;ss&#39;&nbsp;us&#39;</pre> + </p> + <p> + Given a user name, a song, a scrobble count, and a <code>FakeSongService</code>, this function returns a new <code>FakeSongService</code> value with the new data added to the data already there. + </p> + <h3 id="1e89e2737f3e4fe1849f840052f305e2"> + QuickCheck Arbitraries <a href="#1e89e2737f3e4fe1849f840052f305e2">#</a> + </h3> + <p> + In the F# test code I used <a href="https://fscheck.github.io/FsCheck/">FsCheck</a> to get good coverage of the code. For Haskell, I'll use <a href="https://hackage.haskell.org/package/QuickCheck">QuickCheck</a>. + </p> + <p> + Porting the ideas from the F# tests, I define a QuickCheck generator for user names: + </p> + <p> + <pre><span style="color:#2b91af;">alphaNum</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Gen</span>&nbsp;<span style="color:#2b91af;">Char</span> +alphaNum&nbsp;=&nbsp;elements&nbsp;([<span style="color:#a31515;">&#39;a&#39;</span>..<span style="color:#a31515;">&#39;z&#39;</span>]&nbsp;++&nbsp;[<span style="color:#a31515;">&#39;A&#39;</span>..<span style="color:#a31515;">&#39;Z&#39;</span>]&nbsp;++&nbsp;[<span style="color:#a31515;">&#39;0&#39;</span>..<span style="color:#a31515;">&#39;9&#39;</span>]) + +<span style="color:#2b91af;">userName</span>&nbsp;<span style="color:blue;">::</span>&nbsp;<span style="color:blue;">Gen</span>&nbsp;<span style="color:#2b91af;">String</span> +userName&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;len&nbsp;&lt;-&nbsp;choose&nbsp;(1,&nbsp;19) +&nbsp;&nbsp;first&nbsp;&lt;-&nbsp;elements&nbsp;$&nbsp;[<span style="color:#a31515;">&#39;a&#39;</span>..<span style="color:#a31515;">&#39;z&#39;</span>]&nbsp;++&nbsp;[<span style="color:#a31515;">&#39;A&#39;</span>..<span style="color:#a31515;">&#39;Z&#39;</span>] +&nbsp;&nbsp;rest&nbsp;&lt;-&nbsp;vectorOf&nbsp;len&nbsp;alphaNum +&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;$&nbsp;first&nbsp;:&nbsp;rest</pre> + </p> + <p> + It's not that the algorithm only works if usernames are alphanumeric strings that start with a letter and are no longer than twenty characters, but whenever a property is falsified, I'd rather look at a user name like <code>"Yvj0D1I"</code> or <code>"tyD9P1eOqwMMa1Q6u"</code> (which are already bad enough), than something with line breaks and unprintable characters. + </p> + <p> + Working with QuickCheck, it's often <a href="/2019/09/02/naming-newtypes-for-quickcheck-arbitraries">useful to wrap types from the System Under Test in test-specific Arbitrary wrappers</a>: + </p> + <p> + <pre><span style="color:blue;">newtype</span>&nbsp;ValidUserName&nbsp;=&nbsp;ValidUserName&nbsp;{&nbsp;getUserName&nbsp;::&nbsp;String&nbsp;}&nbsp;<span style="color:blue;">deriving</span>&nbsp;(<span style="color:#2b91af;">Show</span>,&nbsp;<span style="color:#2b91af;">Eq</span>) + +<span style="color:blue;">instance</span>&nbsp;<span style="color:blue;">Arbitrary</span>&nbsp;<span style="color:blue;">ValidUserName</span>&nbsp;<span style="color:blue;">where</span> +&nbsp;&nbsp;arbitrary&nbsp;=&nbsp;ValidUserName&nbsp;&lt;$&gt;&nbsp;userName</pre> + </p> + <p> + I also defined a (simpler) <code>Arbitrary</code> instance for <code>Song</code> called <code>AnySong</code>. + </p> + <h3 id="fd6ddacfd3c34430abb30288f5fb37a6"> + A few properties <a href="#fd6ddacfd3c34430abb30288f5fb37a6">#</a> + </h3> + <p> + With <code>FakeSongService</code> in place, I proceeded to add the test code, starting from the top of the F# test code, and translating each as faithfully as possible. The first one is an <a href="https://agileotter.blogspot.com/2008/12/unit-test-ice-breakers.html">Ice Breaker Test</a> that only verifies that the System Under Test exists and doesn't crash when called. + </p> + <p> + <pre>testProperty&nbsp;<span style="color:#a31515;">&quot;No&nbsp;data&quot;</span>&nbsp;$&nbsp;\&nbsp;(ValidUserName&nbsp;un)&nbsp;-&gt;&nbsp;ioProperty&nbsp;$&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;actual&nbsp;&lt;-&nbsp;getRecommendations&nbsp;emptyService&nbsp;un +&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;$&nbsp;<span style="color:blue;">null</span>&nbsp;actual</pre> + </p> + <p> + As I've done since at least 2019, <a href="/2019/03/11/an-example-of-state-based-testing-in-haskell">it seems</a>, I've <a href="/2018/05/07/inlined-hunit-test-lists">inlined test cases as anonymous functions</a>; this time as QuickCheck properties. This one just creates a <code>FakeSongService</code> that contains no data, and asks for recommendations. The expected result is that <code>actual</code> is empty (<code>null</code>), since there's nothing to recommend. + </p> + <p> + A slightly more involved property adds some data to the service before requesting recommendations: + </p> + <p> + <pre>testProperty&nbsp;<span style="color:#a31515;">&quot;One&nbsp;user,&nbsp;some&nbsp;songs&quot;</span>&nbsp;$&nbsp;\ +&nbsp;&nbsp;(ValidUserName&nbsp;user) +&nbsp;&nbsp;(<span style="color:blue;">fmap</span>&nbsp;getSong&nbsp;-&gt;&nbsp;songs) +&nbsp;&nbsp;-&gt;&nbsp;monadicIO&nbsp;$&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;scrobbleCounts&nbsp;&lt;-&nbsp;pick&nbsp;$&nbsp;vectorOf&nbsp;(<span style="color:blue;">length</span>&nbsp;songs)&nbsp;$&nbsp;choose&nbsp;(1,&nbsp;100) +&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;scrobbles&nbsp;=&nbsp;<span style="color:blue;">zip</span>&nbsp;songs&nbsp;scrobbleCounts +&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;srvc&nbsp;=&nbsp;<span style="color:blue;">foldr</span>&nbsp;(<span style="color:blue;">uncurry</span>&nbsp;(scrobble&nbsp;user))&nbsp;emptyService&nbsp;scrobbles + +&nbsp;&nbsp;actual&nbsp;&lt;-&nbsp;run&nbsp;$&nbsp;getRecommendations&nbsp;srvc&nbsp;user + +&nbsp;&nbsp;assertWith&nbsp;(<span style="color:blue;">null</span>&nbsp;actual)&nbsp;<span style="color:#a31515;">&quot;Should&nbsp;be&nbsp;empty&quot;</span></pre> + </p> + <p> + A couple of things are worthy of note. First, the property <a href="/2018/05/14/project-arbitraries-with-view-patterns">uses a view pattern to project a list of songs from a list of Arbitraries</a>, where <code>getSong</code> is the 'getter' that belongs to the <code>AnySong</code> <code>newtype</code> wrapper. + </p> + <p> + I find view patterns quite useful as a declarative way to 'promote' a single <code>Arbitrary</code> instance to a list. In a third property, I take it a step further: + </p> + <p> + <pre>(<span style="color:blue;">fmap</span>&nbsp;getUserName&nbsp;-&gt;&nbsp;NonEmpty&nbsp;users)</pre> + </p> + <p> + This not only turns the singular <code>ValidUserName</code> wrapper into a list, but by projecting it into <code>NonEmpty</code>, the test declares that <code>users</code> is a non-empty list. QuickCheck picks all that up and generates values accordingly. + </p> + <p> + If you're interested in seeing this more advanced view pattern in context, you may consult the Git repository. + </p> + <p> + Secondly, the <code>"One user, some songs"</code> test runs in <code>monadicIO</code>, which I didn't know existed before I wrote these tests. Together with <code>pick</code>, <code>run</code>, and <code>assertWith</code>, <code>monadicIO</code> is defined in <a href="https://hackage.haskell.org/package/QuickCheck/docs/Test-QuickCheck-Monadic.html">Test.QuickCheck.Monadic</a>. It enables you to write properties that run in <code>IO</code>, which these properties need to do, because <code>getRecommendations</code> is <code>IO</code>-bound. + </p> + <p> + There's one more QuickCheck property in the code base, but it mostly repeats techniques already shown here. See the Git repository for all the details, if necessary. + </p> + <h3 id="76fa24be614e460e9af82b2652c82c07"> + Examples <a href="#76fa24be614e460e9af82b2652c82c07">#</a> + </h3> + <p> + In addition to the properties, I also ported the F# examples; that is, 'normal' unit tests. Here's one of them: + </p> + <p> + <pre><span style="color:#a31515;">&quot;One&nbsp;verified&nbsp;recommendation&quot;</span>&nbsp;~:&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;srvc&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;scrobble&nbsp;<span style="color:#a31515;">&quot;ana&quot;</span>&nbsp;(Song&nbsp;2&nbsp;True&nbsp;5)&nbsp;9_9990&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;scrobble&nbsp;<span style="color:#a31515;">&quot;ana&quot;</span>&nbsp;(Song&nbsp;1&nbsp;False&nbsp;5)&nbsp;10&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;scrobble&nbsp;<span style="color:#a31515;">&quot;cat&quot;</span>&nbsp;(Song&nbsp;1&nbsp;False&nbsp;6)&nbsp;10&nbsp;emptyService + +&nbsp;&nbsp;actual&nbsp;&lt;-&nbsp;getRecommendations&nbsp;srvc&nbsp;<span style="color:#a31515;">&quot;cat&quot;</span> + +&nbsp;&nbsp;[Song&nbsp;2&nbsp;True&nbsp;5]&nbsp;@=?&nbsp;actual</pre> + </p> + <p> + This one is straightforward, but as I already discussed when <a href="/2025/04/10/characterising-song-recommendations">characterizing the original code</a>, some of the examples essentially document quirks in the implementation. Here's the relevant test, translated to Haskell: + </p> + <p> + <pre><span style="color:#a31515;">&quot;Only&nbsp;top-rated&nbsp;songs&quot;</span>&nbsp;~:&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;<span style="color:green;">--&nbsp;Scale&nbsp;ratings&nbsp;to&nbsp;keep&nbsp;them&nbsp;less&nbsp;than&nbsp;or&nbsp;equal&nbsp;to&nbsp;10. +</span>&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;srvc&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">foldr</span>&nbsp;(\i&nbsp;-&gt;&nbsp;scrobble&nbsp;<span style="color:#a31515;">&quot;hyle&quot;</span>&nbsp;(Song&nbsp;i&nbsp;True&nbsp;(<span style="color:blue;">toEnum</span>&nbsp;i&nbsp;`div`&nbsp;2))&nbsp;500)&nbsp;emptyService&nbsp;[1..20] + +&nbsp;&nbsp;actual&nbsp;&lt;-&nbsp;getRecommendations&nbsp;srvc&nbsp;<span style="color:#a31515;">&quot;hyle&quot;</span> + +&nbsp;&nbsp;assertBool&nbsp;<span style="color:#a31515;">&quot;Should&nbsp;not&nbsp;be&nbsp;empty&quot;</span>&nbsp;(<span style="color:blue;">not</span>&nbsp;$&nbsp;<span style="color:blue;">null</span>&nbsp;actual) +&nbsp;&nbsp;<span style="color:green;">--&nbsp;Since&nbsp;there&#39;s&nbsp;only&nbsp;one&nbsp;user,&nbsp;but&nbsp;with&nbsp;20&nbsp;songs,&nbsp;the&nbsp;implementation +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;loops&nbsp;over&nbsp;the&nbsp;same&nbsp;songs&nbsp;20&nbsp;times,&nbsp;so&nbsp;400&nbsp;songs&nbsp;in&nbsp;total&nbsp;(with +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;duplicates).&nbsp;Ordering&nbsp;on&nbsp;rating,&nbsp;only&nbsp;the&nbsp;top-rated&nbsp;200&nbsp;remains,&nbsp;that +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;is,&nbsp;those&nbsp;rated&nbsp;5-10.&nbsp;Note&nbsp;that&nbsp;this&nbsp;is&nbsp;a&nbsp;Characterization&nbsp;Test,&nbsp;so&nbsp;not +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;necessarily&nbsp;reflective&nbsp;of&nbsp;how&nbsp;a&nbsp;real&nbsp;recommendation&nbsp;system&nbsp;should&nbsp;work. +</span>&nbsp;&nbsp;assertBool&nbsp;<span style="color:#a31515;">&quot;Should&nbsp;have&nbsp;5+&nbsp;rating&quot;</span>&nbsp;(<span style="color:blue;">all</span>&nbsp;((&gt;=&nbsp;5)&nbsp;.&nbsp;songRating)&nbsp;actual)</pre> + </p> + <p> + This test creates twenty scrobbles for one user: One with a zero rating, two with rating <em>1</em>, two with rating <em>2</em>, and so on, up to a single song with rating <em>10</em>. + </p> + <p> + <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle#interleaved-impurities">The implementation of GetRecommendationsAsync</a> uses these twenty songs to find 'other users' who have these top songs as well. In this case, there's only one user, so for every of those twenty songs, you get the same twenty songs, for a total of 400. + </p> + <p> + There are more unit tests than these. You can see them in the Git repository. + </p> + <h3 id="29a2c5558184424eaaba9db49dc30368"> + Implementation <a href="#29a2c5558184424eaaba9db49dc30368">#</a> + </h3> + <p> + The most direct translation of the C# and F# 'reference implementation' that I could think of was this: + </p> + <p> + <pre>getRecommendations&nbsp;srvc&nbsp;un&nbsp;=&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;<span style="color:green;">--&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users +</span>&nbsp;&nbsp;<span style="color:green;">--&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations +</span> +&nbsp;&nbsp;<span style="color:green;">--&nbsp;Impure +</span>&nbsp;&nbsp;scrobbles&nbsp;&lt;-&nbsp;getTopScrobbles&nbsp;srvc&nbsp;un + +&nbsp;&nbsp;<span style="color:green;">--&nbsp;Pure +</span>&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;scrobblesSnapshot&nbsp;=&nbsp;<span style="color:blue;">take</span>&nbsp;100&nbsp;$&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;scrobbleCount)&nbsp;scrobbles + +&nbsp;&nbsp;recommendationCandidates&nbsp;&lt;-&nbsp;newIORef&nbsp;<span style="color:blue;">[]</span> +&nbsp;&nbsp;forM_&nbsp;scrobblesSnapshot&nbsp;$&nbsp;\scrobble&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">--&nbsp;Impure +</span>&nbsp;&nbsp;&nbsp;&nbsp;otherListeners&nbsp;&lt;-&nbsp;getTopListeners&nbsp;srvc&nbsp;$&nbsp;songId&nbsp;$&nbsp;scrobbledSong&nbsp;scrobble + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">--&nbsp;Pure +</span>&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;otherListenersSnapshot&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;20&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;userScrobbleCount)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;((10_000&nbsp;&lt;=)&nbsp;.&nbsp;userScrobbleCount)&nbsp;otherListeners + +&nbsp;&nbsp;&nbsp;&nbsp;forM_&nbsp;otherListenersSnapshot&nbsp;$&nbsp;\otherListener&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">--&nbsp;Impure +</span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;otherScrobbles&nbsp;&lt;-&nbsp;getTopScrobbles&nbsp;srvc&nbsp;$&nbsp;userName&nbsp;otherListener + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">--&nbsp;Pure +</span>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;otherScrobblesSnapshot&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">take</span>&nbsp;10&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;songRating&nbsp;.&nbsp;scrobbledSong)&nbsp;$ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">filter</span>&nbsp;(songHasVerifiedArtist&nbsp;.&nbsp;scrobbledSong)&nbsp;otherScrobbles + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;forM_&nbsp;otherScrobblesSnapshot&nbsp;$&nbsp;\otherScrobble&nbsp;-&gt;&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;song&nbsp;=&nbsp;scrobbledSong&nbsp;otherScrobble +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;modifyIORef&nbsp;recommendationCandidates&nbsp;(song&nbsp;:) + +&nbsp;&nbsp;recommendations&nbsp;&lt;-&nbsp;readIORef&nbsp;recommendationCandidates +&nbsp;&nbsp;<span style="color:green;">--&nbsp;Pure +</span>&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;$&nbsp;<span style="color:blue;">take</span>&nbsp;200&nbsp;$&nbsp;sortOn&nbsp;(Down&nbsp;.&nbsp;songRating)&nbsp;recommendations</pre> + </p> + <p> + In order to mirror <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle#interleaved-impurities">the original implementation</a> as closely as possible, I declare <code>recommendationCandidates</code> as an <a href="https://hackage.haskell.org/package/base/docs/Data-IORef.html">IORef</a> so that I can incrementally add to it as the action goes through its nested loops. Notice the <code>modifyIORef</code> towards the end of the code listing, which adds a single song to the list. + </p> + <p> + Once all the looping is done, the action uses <code>readIORef</code> to pull the <code>recommendations</code> out of the <code>IORef</code>. + </p> + <p> + As you can see, I also ported the comments from the original C# code. + </p> + <p> + I don't consider this idiomatic Haskell code, but the goal in this article was to mirror the C# code as closely as possible. Once I start refactoring, you'll see some more idiomatic implementations. + </p> + <h3 id="8192c1e94add4e56b95dad78d4eda818"> + Conclusion <a href="#8192c1e94add4e56b95dad78d4eda818">#</a> + </h3> + <p> + Together with the previous two articles in this article series, this establishes a baseline from which I can refactor the code. While we might consider the original C# code idiomatic, this port to Haskell isn't. It is, on the other hand, similar enough to both its C# and F# peers that we can compare and contrast all three. + </p> + <p> + Particularly two design choices make this Haskell implementation less than idiomatic. One is the use of <code>IORef</code> to update a list of songs. The other is using a type class to model an external dependency. + </p> + <p> + As I cover various alternative architectures in this article series, you'll see how to get rid of both. + </p> + <p> + <strong>Next:</strong> <a href="/2025/04/28/song-recommendations-as-an-impureim-sandwich">Song recommendations as an Impureim Sandwich</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/04/21/porting-song-recommendations-to-haskell + + + + Porting song recommendations to F# + https://blog.ploeh.dk/2025/04/14/porting-song-recommendations-to-f/ + Mon, 14 Apr 2025 08:54:00 UTC + + + +<div id="post"> + <p> + <em>A C# code base translated to F#.</em> + </p> + <p> + This article is part of a <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">larger article series</a> that examines variations of how to take on a non-trivial problem using <a href="/2018/11/19/functional-architecture-a-definition">functional architecture</a>. In the <a href="/2025/04/10/characterising-song-recommendations">previous article</a> we established a baseline C# code base. Future articles are going to use that C# code base as a starting point for refactored code. On the other hand, I also want to demonstrate what such solutions may look like in languages like <a href="https://fsharp.org/">F#</a> or <a href="https://www.haskell.org/">Haskell</a>. In this article, you'll see how to port the C# baseline to F#. + </p> + <p> + The code shown in this article is from the <em>fsharp-port</em> branch of the accompanying Git repository. + </p> + <h3 id="075d8dc2f6ad4baca4f815137193f814"> + Data structures <a href="#075d8dc2f6ad4baca4f815137193f814">#</a> + </h3> + <p> + We may start by defining the required data structures. All are going to be <a href="https://learn.microsoft.com/dotnet/fsharp/language-reference/records">records</a>. + </p> + <p> + <pre><span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">User</span>&nbsp;=&nbsp;{&nbsp;UserName&nbsp;:&nbsp;<span style="color:#2b91af;">string</span>;&nbsp;TotalScrobbleCount&nbsp;:&nbsp;<span style="color:#2b91af;">int</span>&nbsp;}</pre> + </p> + <p> + Just like the equivalent C# code, a <code>User</code> is just a <code>string</code> and an <code>int</code>. + </p> + <p> + When creating new values, record syntax can sometimes be awkward, so I also define a curried function to create <code>User</code> values: + </p> + <p> + <pre><span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">user</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">totalScrobbleCount</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;{&nbsp;UserName&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>;&nbsp;TotalScrobbleCount&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">totalScrobbleCount</span>&nbsp;}</pre> + </p> + <p> + Likewise, I define <code>Song</code> and <code>Scrobble</code> in the same way: + </p> + <p> + <pre><span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">Song</span>&nbsp;=&nbsp;{&nbsp;Id&nbsp;:&nbsp;<span style="color:#2b91af;">int</span>;&nbsp;IsVerifiedArtist&nbsp;:&nbsp;<span style="color:#2b91af;">bool</span>;&nbsp;Rating&nbsp;:&nbsp;<span style="color:#2b91af;">byte</span>&nbsp;} +<span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">song</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">isVerfiedArtist</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">rating</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;{&nbsp;Id&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">id</span>;&nbsp;IsVerifiedArtist&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">isVerfiedArtist</span>;&nbsp;Rating&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">rating</span>&nbsp;} + +<span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">Scrobble</span>&nbsp;=&nbsp;{&nbsp;Song&nbsp;:&nbsp;<span style="color:#2b91af;">Song</span>;&nbsp;ScrobbleCount&nbsp;:&nbsp;<span style="color:#2b91af;">int</span>&nbsp;} +<span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">scrobble</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">song</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbleCount</span>&nbsp;=&nbsp;{&nbsp;Song&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">song</span>;&nbsp;ScrobbleCount&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbleCount</span>&nbsp;}</pre> + </p> + <p> + To be honest, I only use those curried functions sparingly, so they're somewhat redundant. Perhaps I should consider getting rid of them. For now, however, they stay. + </p> + <p> + Since I'm moving all the code to F#, I also have to translate the interface. + </p> + <p> + <pre><span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">SongService</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">abstract</span>&nbsp;<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>&nbsp;:&nbsp;songId&nbsp;:&nbsp;<span style="color:#2b91af;">int</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">User</span>&gt;&gt; +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">abstract</span>&nbsp;<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>&nbsp;:&nbsp;userName&nbsp;:&nbsp;<span style="color:#2b91af;">string</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#2b91af;">Task</span>&lt;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;<span style="color:#2b91af;">Scrobble</span>&gt;&gt;</pre> + </p> + <p> + The syntax is different from C#, but otherwise, this is the same interface. + </p> + <h3 id="4102e5ed7f394f48ac28f4b4a8af55e8"> + Implementation <a href="#4102e5ed7f394f48ac28f4b4a8af55e8">#</a> + </h3> + <p> + Those are all the supporting types required to implement the <code>RecommendationsProvider</code>. This is the most direct translation of the C# code that I could think of: + </p> + <p> + <pre><span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">RecommendationsProvider</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">songService</span>&nbsp;:&nbsp;<span style="color:#2b91af;">SongService</span>)&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">member</span>&nbsp;_.<span style="font-weight:bold;color:#74531f;">GetRecommendationsAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>&nbsp;=&nbsp;<span style="color:blue;">task</span>&nbsp;{ +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;1.&nbsp;Get&nbsp;user&#39;s&nbsp;own&nbsp;top&nbsp;scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;2.&nbsp;Get&nbsp;other&nbsp;users&nbsp;who&nbsp;listened&nbsp;to&nbsp;the&nbsp;same&nbsp;songs</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;3.&nbsp;Get&nbsp;top&nbsp;scrobbles&nbsp;of&nbsp;those&nbsp;users</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;4.&nbsp;Aggregate&nbsp;the&nbsp;songs&nbsp;into&nbsp;recommendations</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;=&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobblesSnapshot</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.ScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;100 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>&nbsp;=&nbsp;<span style="color:#2b91af;">ResizeArray</span>&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">for</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobblesSnapshot</span>&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListeners</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobble</span>.Song.Id + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListenersSnapshot</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListeners</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount&nbsp;&gt;=&nbsp;10_000) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">u</span>.TotalScrobbleCount) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;20 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">for</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListener</span>&nbsp;<span style="color:blue;">in</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListenersSnapshot</span>&nbsp;<span style="color:blue;">do</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Impure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let!</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">songService</span>.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherListener</span>.UserName + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobblesSnapshot</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobbles</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.IsVerifiedArtist) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;10 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">otherScrobblesSnapshot</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">List</span>.<span style="color:#74531f;">map</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Song) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span>.<span style="font-weight:bold;color:#74531f;">AddRange</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:green;">//&nbsp;Pure</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendations</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendationCandidates</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sortByDescending</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">s</span>.Rating) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">truncate</span>&nbsp;200 +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;:&gt;&nbsp;<span style="color:#2b91af;">IReadOnlyCollection</span>&lt;_&gt; + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">return</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">recommendations</span>&nbsp;}</pre> + </p> + <p> + As you can tell, I've kept the comments from <a href="https://tyrrrz.me/blog/pure-impure-segregation-principle">the original</a>, too. + </p> + <h3 id="efa00ed5a67641e18b6fec772f2e6aa7"> + Test Double <a href="#efa00ed5a67641e18b6fec772f2e6aa7">#</a> + </h3> + <p> + In <a href="/2025/04/10/characterising-song-recommendations">the previous article</a>, I'd written the <a href="http://xunitpatterns.com/Fake%20Object.html">Fake</a> <code>SongService</code> in C#. Since, in this article, I'm translating everything to F#, I need to translate the Fake, too. + </p> + <p> + <pre><span style="color:blue;">type</span>&nbsp;<span style="color:#2b91af;">FakeSongService</span>&nbsp;()&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span>&nbsp;=&nbsp;<span style="color:#2b91af;">ConcurrentDictionary</span>&lt;<span style="color:#2b91af;">int</span>,&nbsp;<span style="color:#2b91af;">Song</span>&gt;&nbsp;() +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span>&nbsp;=&nbsp;<span style="color:#2b91af;">ConcurrentDictionary</span>&lt;<span style="color:#2b91af;">string</span>,&nbsp;<span style="color:#2b91af;">ConcurrentDictionary</span>&lt;<span style="color:#2b91af;">int</span>,&nbsp;<span style="color:#2b91af;">int</span>&gt;&gt;&nbsp;() + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">interface</span>&nbsp;<span style="color:#2b91af;">SongService</span>&nbsp;<span style="color:blue;">with</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">member</span>&nbsp;_.<span style="font-weight:bold;color:#74531f;">GetTopListenersAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songId</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">listeners</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">filter</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>.Value.<span style="font-weight:bold;color:#74531f;">ContainsKey</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songId</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">map</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#74531f;">user</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>.Key&nbsp;(<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">sum</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>.Value.Values)) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Task</span>.<span style="font-weight:bold;color:#74531f;">FromResult</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">listeners</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">member</span>&nbsp;_.<span style="font-weight:bold;color:#74531f;">GetTopScrobblesAsync</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span>.<span style="font-weight:bold;color:#74531f;">GetOrAdd</span>(<span style="font-weight:bold;color:#1f377f;">userName</span>,&nbsp;<span style="color:#2b91af;">ConcurrentDictionary</span>&lt;_,&nbsp;_&gt;&nbsp;()) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">map</span>&nbsp;(<span style="color:blue;">fun</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#74531f;">scrobble</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span>[<span style="font-weight:bold;color:#1f377f;">kvp</span>.Key]&nbsp;<span style="font-weight:bold;color:#1f377f;">kvp</span>.Value) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#2b91af;">Seq</span>.<span style="color:#74531f;">toList</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">Task</span>.<span style="font-weight:bold;color:#74531f;">FromResult</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span> + +&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">member</span>&nbsp;_.<span style="font-weight:bold;color:#74531f;">Scrobble</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">userName</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">song</span>&nbsp;:&nbsp;<span style="color:#2b91af;">Song</span>,&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbleCount</span>)&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">let</span>&nbsp;<span style="color:#74531f;">addScrobbles</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;:&nbsp;<span style="color:#2b91af;">ConcurrentDictionary</span>&lt;_,&nbsp;_&gt;)&nbsp;= +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>.<span style="font-weight:bold;color:#74531f;">AddOrUpdate</span>&nbsp;( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">song</span>.Id, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbleCount</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">fun</span>&nbsp;_&nbsp;<span style="font-weight:bold;color:#1f377f;">oldCount</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">oldCount</span>&nbsp;+&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbleCount</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#74531f;">ignore</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span> + +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">users</span>.<span style="font-weight:bold;color:#74531f;">AddOrUpdate</span>&nbsp;( +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">userName</span>, +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:#2b91af;">ConcurrentDictionary</span>&lt;_,&nbsp;_&gt; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[&nbsp;<span style="color:#2b91af;">KeyValuePair</span>.<span style="font-weight:bold;color:#74531f;">Create</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">song</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbleCount</span>)&nbsp;], +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="color:blue;">fun</span>&nbsp;_&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="color:#74531f;">addScrobbles</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">scrobbles</span>) +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;|&gt;&nbsp;<span style="color:#74531f;">ignore</span> +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp; +&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<span style="font-weight:bold;color:#1f377f;">songs</span>.<span style="font-weight:bold;color:#74531f;">AddOrUpdate</span>&nbsp;(<span style="font-weight:bold;color:#1f377f;">song</span>.Id,&nbsp;<span style="font-weight:bold;color:#1f377f;">song</span>,&nbsp;<span style="color:blue;">fun</span>&nbsp;_&nbsp;_&nbsp;<span style="color:blue;">-&gt;</span>&nbsp;<span style="font-weight:bold;color:#1f377f;">song</span>)&nbsp;|&gt;&nbsp;<span style="color:#74531f;">ignore</span></pre> + </p> + <p> + Apart from the code shown here, only minor changes were required for the tests, such as using those curried creation functions instead of constructors, a cast to <code>SongService</code>, and a few other non-behavioural things like that. All tests still pass, so I consider this a faithful translation of the C# code base. + </p> + <h3 id="28c7110824e941098e791aa66e2ed5c4"> + Conclusion <a href="#28c7110824e941098e791aa66e2ed5c4">#</a> + </h3> + <p> + This article does more groundwork. Since it may be illuminating to see one problem represented in more than one programming language, I present it in both C#, F#, and Haskell. The next article does exactly that: Translates this F# code to Haskell. Once all three bases are established, we can start introducing solution variations. + </p> + <p> + If you don't care about the Haskell examples, you can always go back to the <a href="/2025/04/07/alternative-ways-to-design-with-functional-programming">first article in this article series</a> and use the table of contents to jump to the next C# example. + </p> + <p> + <strong>Next:</strong> <a href="/2025/04/21/porting-song-recommendations-to-haskell">Porting song recommendations to Haskell</a>. + </p> +</div><hr> + This blog is totally free, but if you like it, please consider <a href="https://blog.ploeh.dk/support">supporting it</a>. + Mark Seemann + https://blog.ploeh.dk/2025/04/14/porting-song-recommendations-to-f + + + + + diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..e34bc4c --- /dev/null +++ b/src/error.rs @@ -0,0 +1,22 @@ +use std::fmt::Display; + +use xml::reader::Error; + +#[derive(Debug)] +pub enum TrsError { + XmlParseError(String), + XmlRsError(Error), +} + +impl Display for TrsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + TrsError::XmlParseError(msg) => format!("XML Parse Error: {}", msg), + TrsError::XmlRsError(err) => format!("XML Reader Error: {}", err), + } + ) + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..63169d9 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,26 @@ +use error::TrsError; +use xml::ParserConfig; +pub mod error; +pub mod parser; + +fn main() -> Result<(), TrsError> { + let bytes = include_bytes!("../sample/rss2.xml"); + let xml_source_stream = ParserConfig::new() + .ignore_invalid_encoding_declarations(true) + .create_reader(&bytes[..]); + let rss_channel = parser::parse_rss_channel(xml_source_stream)?; + + println!("{}", rss_channel.title); + println!("{}", rss_channel.link); + println!("{}", rss_channel.description); + for article in &rss_channel.articles { + println!("{} {:^50} {:<}", article.date, article.title, article.link); + } + + println!( + "There are {} articles in the channel.", + rss_channel.articles.len() + ); + + Ok(()) +} diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..0adb8a5 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,221 @@ +use std::io::Read; + +use xml::{reader::XmlEvent, EventReader}; + +use crate::error::TrsError; + +pub struct RssChannel { + pub title: String, + pub link: String, + pub description: String, + pub articles: Vec
, +} + +impl RssChannel { + fn new() -> Self { + RssChannel { + title: String::new(), + link: String::new(), + description: String::new(), + articles: Vec::new(), + } + } + + fn update_channel_field(&mut self, field: &XmlTagField, value: String) -> Result<(), TrsError> { + let last_article = self.articles.last_mut(); + let no_item_error = || { + TrsError::XmlParseError(format!( + "No item found to update field <{}>", + field.hierarchical_tag + )) + }; + + match field.field { + XmlField::ArticleTitle => self.title = value, + XmlField::ArticleLink => self.link = value, + XmlField::ArticleDescription => self.description = value, + XmlField::ItemTitle => last_article.ok_or_else(no_item_error)?.title = value, + XmlField::ItemLink => last_article.ok_or_else(no_item_error)?.link = value, + XmlField::ItemPubDate => last_article.ok_or_else(no_item_error)?.date = value, + } + + Ok(()) + } +} + +pub struct Article { + pub title: String, + pub link: String, + pub date: String, +} + +impl Article { + fn new() -> Self { + Article { + title: String::new(), + link: String::new(), + date: String::new(), + } + } +} + +enum XmlField { + ItemTitle, + ItemLink, + ItemPubDate, + ArticleTitle, + ArticleLink, + ArticleDescription, +} + +struct XmlTagField { + hierarchical_tag: &'static str, + tag: &'static str, + field: XmlField, +} + +impl XmlTagField { + const fn mapping(hierarchical_tag: &'static str, tag: &'static str, field: XmlField) -> Self { + XmlTagField { + hierarchical_tag, + tag, + field, + } + } + + fn corresponding_field(hierarchical_tag: &str) -> Option<&'static XmlTagField> { + for field in FIELD_TAG_MAPPINGS.iter() { + if field.hierarchical_tag == hierarchical_tag { + return Some(field); + } + } + + None + } +} + +const FIELD_TAG_MAPPINGS: [XmlTagField; 6] = [ + XmlTagField::mapping("title", "title", XmlField::ArticleTitle), + XmlTagField::mapping("link", "link", XmlField::ArticleLink), + XmlTagField::mapping("description", "description", XmlField::ArticleDescription), + XmlTagField::mapping("item > title", "title", XmlField::ItemTitle), + XmlTagField::mapping("item > link", "link", XmlField::ItemLink), + XmlTagField::mapping("item > pubDate", "pubDate", XmlField::ItemPubDate), +]; + +pub fn parse_rss_channel( + xml_source_stream: EventReader, +) -> Result { + let mut channel = RssChannel::new(); + let mut tag_prefix = ""; + let mut current_field: Option<&XmlTagField> = None; + for e in xml_source_stream { + match e { + Ok(XmlEvent::StartElement { name, .. }) => match name.local_name.as_str() { + "item" => { + tag_prefix = "item > "; + channel.articles.push(Article::new()); + } + tag => { + let None = current_field else { + let current_field_name = current_field.unwrap(); + return Err(TrsError::XmlParseError(format!( + "Unexpected <{}> start tag without closing existing tag <{}>", + tag, current_field_name.hierarchical_tag + ))); + }; + + let tag_name_with_prefix = format!("{}{}", tag_prefix, tag); + current_field = XmlTagField::corresponding_field(&tag_name_with_prefix); + } + }, + Ok(XmlEvent::EndElement { name }) => match name.local_name.as_str() { + "item" => { + let None = current_field else { + let current_field_name = current_field.unwrap(); + return Err(TrsError::XmlParseError(format!( + "Unexpected end tag without closing field {}", + current_field_name.hierarchical_tag + ))); + }; + tag_prefix = ""; + } + tag => { + if let Some(field) = current_field.take() { + if field.tag == tag { + current_field = None; + } else { + return Err(TrsError::XmlParseError(format!( + "Unexpected end tag, expected ", + tag, field.hierarchical_tag + ))); + } + } + } + }, + Ok(XmlEvent::Characters(data)) => { + if let Some(field) = current_field { + let err = channel.update_channel_field(field, data); + if let Err(e) = err { + eprintln!("Error updating channel field: {}", e); + return Err(e); + } + } + } + Err(e) => { + eprintln!("Error parsing XML: {}", e); + return Err(TrsError::XmlRsError(e)); + } + _ => {} + } + } + + Ok(channel) +} + +#[cfg(test)] +mod tests { + use super::*; + use xml::ParserConfig; + + macro_rules! validate_sample { + ($test_name:ident, $file_name:literal, $title:literal, $link:literal, $description: literal, $article_count: literal) => { + #[test] + fn $test_name() { + let bytes = include_bytes!(concat!("../sample/", $file_name)); + let xml_source_stream = ParserConfig::new() + .ignore_invalid_encoding_declarations(true) + .create_reader(&bytes[..]); + let rss_channel = parse_rss_channel(xml_source_stream).unwrap(); + + assert_eq!(rss_channel.title, $title); + assert_eq!(rss_channel.link, $link); + assert_eq!(rss_channel.description, $description); + assert_eq!(rss_channel.articles.len(), $article_count); + for article in &rss_channel.articles { + assert!(!article.title.is_empty()); + assert!(!article.link.is_empty()); + assert!(!article.date.is_empty()); + } + } + }; + } + + validate_sample!( + sample1, + "rss.xml", + "Bryce Vandegrift's Website", + "https://brycev.com/", + "Updates to Bryce Vandegrift's blog", + 28 + ); + + validate_sample!( + sample2, + "rss2.xml", + "ploeh blog", + "https://blog.ploeh.dk", + "danish software design", + 10 + ); +}