Ended up going with Sprig, which seems to work pretty well. Open to ideas on how to improve this, especially if there's some better options for performance.
{% set colour = colour ?? '' %}
{% set addedToCart = success is defined %}
{% set variants = craft.variants().productId(productId).relatedTo(colour).all() %}
{% set colours = craft.categories().group('colours').all() %}
<form sprig s-method="post" s-action="commerce/cart/update-cart">
<input type="hidden" name="qty" value="1">
<fieldset>
<legend>Colour</legend>
<div>
{% for colourOption in colours %}
<label>
<input sprig type="radio" name="colour" value="{{ colourOption.id }}">
<p id="colour-label" class="sr-only">
{{ colourOption.title }}
</p>
<span class="h-8 w-8 bg-{{ colourOption.slug }}-500 border-gray-200 rounded-full p-2 {{ colour == colourOption.id ? "ring-4 ring-gold-500" : "ring-2 ring-gray-200" }}"></span>
</label>
{% endfor %}
</div>
</fieldset>
<fieldset class="mb-8">
<legend>
Size
</legend>
<div>
{% for variant in variants %}
<label>
<input type="radio" name="purchasableId" value="{{ variant.id }}">
{{ variant.title }}
</label>
{% endfor %}
</fieldset>
<button type="submit">Add to cart</button>
{% if addedToCart and success %}
<div>{{ product.title }}</span> was added to your cart!</div>
{% endif %}
</form>