time_duration inefficient compared to ptime and gregorian::date

Compile and run the benchmark program using the listed Boost date_time headers, then compare the sort timings for time_duration, int64_t, date, and ptime. Trace the time_duration comparison path to identify why it is slower; done means explaining the cause and documenting or addressing the reported inefficiency.

compile this program:

#include <boost/date_time/posix_time/ptime.hpp>
#include <boost/date_time/posix_time/posix_time_duration.hpp>
#include <iostream>
#include <vector>
#include <chrono>

using namespace boost::posix_time;
using namespace boost::gregorian;
using namespace std;

int main(int argc, char *argv[])
{
  const int64_t count = 1000000;

  {
    vector<time_duration> v;
    v.reserve(count);
    for (int64_t i=0; i<count; ++i) v.push_back(seconds(i));

    auto start = chrono::high_resolution_clock::now();
    sort(v.begin(), v.end());
    auto finish = std::chrono::high_resolution_clock::now();
    cout << "time_duration sort: " << chrono::duration<double>(finish - start).count() << "s\n";
  }

  {
    vector<int64_t> v;
    v.reserve(count);
    for (int64_t i=0; i<count; ++i) v.push_back(i);

    auto start = chrono::high_resolution_clock::now();
    sort(v.begin(), v.end());
    auto finish = std::chrono::high_resolution_clock::now();
    cout << "int64_t sort: " << chrono::duration<double>(finish - start).count() << "s\n";
  }

  {
    vector<date> v;
    v.reserve(count);
    date d(1400,1,1);
    for (int64_t i=0; i<count; ++i) {
      v.push_back(d);
      d+=date_duration(1);
    }

    auto start = chrono::high_resolution_clock::now();
    sort(v.begin(), v.end());
    auto finish = std::chrono::high_resolution_clock::now();
    cout << "date sort: " << chrono::duration<double>(finish - start).count() << "s\n";
  }

  {
    vector<ptime> v;
    v.reserve(count);
    ptime p(date(1400,1,1));
    for (int64_t i=0; i<count; ++i) {
      v.push_back(ptime(p));
      p+=hours(5)+minutes(4)+seconds(3);
    }

    auto start = chrono::high_resolution_clock::now();
    sort(v.begin(), v.end());
    auto finish = std::chrono::high_resolution_clock::now();
    cout << "ptime sort: " << chrono::duration<double>(finish - start).count() << "s\n";
  }

  return 0;
}

program output on my machine:

time_duration sort: 3.60364s
int64_t sort: 0.560309s
date sort: 0.610657s
ptime sort: 1.36204s

the question is, what is so special about time_duration making it so unbelievably inefficient? it is even more inefficient than ptime!