<!-- entry -->
        {% if entry is defined %}

            {% for siteForEntry in entry.getSupportedSites %}

                {% set site = craft.app.getSites.getSiteById(siteForEntry.siteId) %}

                {% if site.id != entry.siteId %}

                    {% set entryForOtherSite = craft.entries.id(entry.id).site(site).one() %}

                    {% if entryForOtherSite is not null %}

                        <a href="{{ entryForOtherSite.getUrl() }}">{{ site.name }}</a>

                    {% endif %}

                {% endif %}
                   
            {% endfor %}

        <!-- category -->
        {% elseif category is defined %}

            {% set sites = craft.app.sites.getAllSites() %}

            {% for site in sites %}
                <li><a href="{{ site.baseUrl }}blog/category/{{craft.request.getLastSegment()}}">{{ site.name }}</a></li>
            {% endfor %}

        <!-- index and tags -->
        {% else %}

            {% set sites = craft.app.sites.getAllSites() %}

            {% for site in sites %}

                <li><a href="{{ site.baseUrl }}{% for segment in craft.request.getSegments() %}{{segment}}/{% endfor %}">{{ site.name }}</a></li>
                
            {% endfor %}

        {% endif %}

        <!-- entry -->
        {% if entry is defined %}

            {% for siteForEntry in entry.getSupportedSites %}

                {% set site = craft.app.getSites.getSiteById(siteForEntry.siteId) %}

                {% if site.id != entry.siteId %}

                    {% set entryForOtherSite = craft.entries.id(entry.id).site(site).one() %}

                    {% if entryForOtherSite is not null %}

                        <li><a href="{{ entryForOtherSite.getUrl() }}">{{ site.name }}</a></li>

                    {% endif %}

                {% endif %}
                   
            {% endfor %}

        <!-- category -->
        {% elseif category is defined %}

            {% set currentSite = craft.app.sites.currentSite %}

            {% set sites = craft.app.sites.getAllSites() %}

            {% for site in sites %}

                {% if site.id != currentSite.id %} 

                    <li><a href="{{ site.baseUrl }}blog/category/{{craft.request.getLastSegment()}}">{{ site.name }}</a></li>

                {% endif %}

            {% endfor %}

        <!-- index and tags -->
        {% else %}

            {% set currentSite = craft.app.sites.currentSite %}

            {% set sites = craft.app.sites.getAllSites() %}

            {% for site in sites %}

                {% if site.id != currentSite.id %}

                    <li><a href="{{ site.baseUrl }}{% for segment in craft.request.getSegments() %}{{segment}}/{% endfor %}">{{ site.name }}</a></li>

                {% endif %}
                
            {% endfor %}

        {% endif %}

This is a lot closer. The switcher now will not display the link if the entry is not enable for other site. The index part isn't perfect. If you go to /blog and change the language to for example german it will work: /blog/de. But then if you switch the language again to for example french it will go to /fr/de/blog

            <!-- entry -->
        {% if entry is defined %}

            {% for siteForEntry in entry.getSupportedSites %}

                {% set site = craft.app.getSites.getSiteById(siteForEntry.siteId) %}

                {% if site.id != entry.siteId %}

                    {% set entryForOtherSite = craft.entries.id(entry.id).site(site).one() %}

                    {% if entryForOtherSite is not null %}

                        <a href="{{ entryForOtherSite.getUrl() }}">{{ site.language }}</a>

                    {% endif %}

                {% endif %}
                   
            {% endfor %}

        <!-- index -->
        {% else %}

            {% set localizedUrlsArray = seomatic.helper.getLocalizedUrls() %}

            {% for localizedUrls in localizedUrlsArray %}

                <a href="{{ localizedUrls['url'] }}"> {{ localizedUrls['language'] }}</a>

            {% endfor %}

        {% endif %}

I think I might finally have it working. If someone has a more elegant way to handle the index, tag and category I would love to see it!

        <!-- entry -->
        {% if entry is defined %}

            {% for siteForEntry in entry.getSupportedSites %}

                {% set site = craft.app.getSites.getSiteById(siteForEntry.siteId) %}

                {% if site.id != entry.siteId %}

                    {% set entryForOtherSite = craft.entries.id(entry.id).site(site).one() %}

                    {% if entryForOtherSite is not null %}

                        <a href="{{ entryForOtherSite.getUrl() }}">{{ site.name }}</a>

                    {% endif %}

                {% endif %}
                   
            {% endfor %}

        <!-- category -->
        {% elseif category is defined %}

            {% set sites = craft.app.sites.getAllSites() %}

            {% for site in sites %}
                <li><a href="{{ site.baseUrl }}blog/category/{{craft.request.getLastSegment()}}">{{ site.name }}</a></li>
            {% endfor %}

        <!-- index and tags -->
        {% else %}

            {% set sites = craft.app.sites.getAllSites() %}

            {% for site in sites %}

                <li><a href="{{ site.baseUrl }}{% for segment in craft.request.getSegments() %}{{segment}}/{% endfor %}">{{ site.name }}</a></li>
                
            {% endfor %}

        {% endif %}