I think I might finally have it working. If someone has a more elegant way to handle the index, tag and category I would love to see it! 

		  	<!-- entry -->
			{% if entry is defined %}

				{% for siteForEntry in entry.getSupportedSites %}

				    {% set site = craft.app.getSites.getSiteById(siteForEntry.siteId) %}

					{% if site.id != entry.siteId %}

					    {% set entryForOtherSite = craft.entries.id(entry.id).site(site).one() %}

				        {% if entryForOtherSite is not null %}

				        	<a href="{{ entryForOtherSite.getUrl() }}">{{ site.name }}</a>

					    {% endif %}

					{% endif %}
					   
				{% endfor %}

			<!-- category -->
			{% elseif category is defined %}

				{% set sites = craft.app.sites.getAllSites() %}

				{% for site in sites %}
				    <li><a href="{{ site.baseUrl }}blog/category/{{craft.request.getLastSegment()}}">{{ site.name }}</a></li>
				{% endfor %}

			<!-- index and tags -->
			{% else %}

				{% set sites = craft.app.sites.getAllSites() %}

				{% for site in sites %}

				    <li><a href="{{ site.baseUrl }}{% for segment in craft.request.getSegments() %}{{segment}}/{% endfor %}">{{ site.name }}</a></li>
				    
				{% endfor %}

			{% endif %}