I think I might finally have it working. If someone has a more elegant way to handle the index, tag and category I would love to see it!
<!-- entry -->
{% if entry is defined %}
{% for siteForEntry in entry.getSupportedSites %}
{% set site = craft.app.getSites.getSiteById(siteForEntry.siteId) %}
{% if site.id != entry.siteId %}
{% set entryForOtherSite = craft.entries.id(entry.id).site(site).one() %}
{% if entryForOtherSite is not null %}
<li><a href="{{ entryForOtherSite.getUrl() }}">{{ site.name }}</a></li>
{% endif %}
{% endif %}
{% endfor %}
<!-- category -->
{% elseif category is defined %}
{% set currentSite = craft.app.sites.currentSite %}
{% set sites = craft.app.sites.getAllSites() %}
{% for site in sites %}
{% if site.id != currentSite.id %}
<li><a href="{{ site.baseUrl }}blog/category/{{craft.request.getLastSegment()}}">{{ site.name }}</a></li>
{% endif %}
{% endfor %}
<!-- index and tags -->
{% else %}
{% set currentSite = craft.app.sites.currentSite %}
{% set sites = craft.app.sites.getAllSites() %}
{% for site in sites %}
{% if site.id != currentSite.id %}
<li><a href="{{ site.baseUrl }}{% for segment in craft.request.getSegments() %}{{segment}}/{% endfor %}">{{ site.name }}</a></li>
{% endif %}
{% endfor %}
{% endif %}