/*
 * x86_rdtscp1.c - test x86 RDTSCP instruction to get the current
 * CPU number
 *
 * Compile with:
 * $ gcc -std=gnu17 -m64 -Wall x86_rdtscp1.c -lpthread
 *
 * Written by Roland Mainz <roland.mainz@nrubsig.org>
 *
 */

#define _XOPEN_SOURCE 700
#define _GNU_SOURCE 1

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>

#include <pthread.h>
#include <sched.h>

#include <x86intrin.h>


int main(int ac, char *av[])
{
	(void)puts("#start.");

	unsigned int A = 0;
	unsigned long long retval = 0;
	int cpu;

	/*
	 * The x86 rdtscp instruction and |sched_getcpu()| syscall can
	 * return different results when the scheduler reschedules the
	 * running thread, usually at the syscall boundary.
	 * 
	 * Notes:
	 * - Using the FIFO scheduler can reduce the instances where
	 * this happens, but even then the scheduler can get active:
	 * $ time chrt --fifo 20 ksh93 -c 'while true ; do \
	 * 	x="$(/home/rmainz/tmp/x86_rdtscp/a.out)" ; \
	 * 	[[ "$x" == *match* ]] && break ; done ; \
	 * 	printf "res=%s\n" "$x"'
	 * res=#start.
	 * _rdtscp:        A=7, retval=43db24d4804f4e
	 * sched_getcpu(): cpu=3
	 * # mismatch A!=cpu
	 * #done.
	 * real    5m27.782s
	 * user    3m53.786s
	 * sys     1m40.138s
	 *
	 * - The only useable solution is to pin the thread to one
	 * cpu strand, as shown in the |USE_FIXED_CPU_NUMBER| codepath
	 * below...
	 *
	 * - Read https://unix4lyfe.org/benchmarking/
	 *
	 * - On some machines the cpu (strand) number is always |0|,
	 * independent from which CPU/strand the thread is really
	 * executing on... ;-(
	 * Current (2022-10-17) working theory is that this is the CPU ID
	 * of timer (the RDTSCP instruction returns a timer value!!) and
	 * the specific CPU only has one timer...
	 */

#define USE_FIXED_CPU_NUMBER 1
#if USE_FIXED_CPU_NUMBER
	cpu_set_t cpuset;
	pthread_t thread;

	thread = pthread_self();
	cpu = sched_getcpu();

	CPU_ZERO(&cpuset);
	CPU_SET(cpu, &cpuset);

	if (pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset) == 0) {
		(void)printf("# pthread_setaffinity_np(), "
			"thread fixed to cpu %d\n",
			cpu);
	}
	else {
		/*
		 * If you specify a CPU number beyond the maximum number
		 * of CPUs, then |pthread_setaffinity_np()| can have a
		 * |errno == 0|
		 */
		perror("pthread_setaffinity_np");
	}
#endif /* USE_FIXED_CPU_NUMBER */

	retval = _rdtscp(&A);
	cpu = sched_getcpu();

	(void)printf("_rdtscp:\tA=%x, ", A);
	(void)printf("retval=%llx\n", retval);
	(void)printf("sched_getcpu():\tcpu=%x\n", cpu);
	
	if (cpu != A)
		(void)puts("# mismatch A!=cpu");

	(void)puts("#done.");

	return EXIT_SUCCESS;
}