compilation_basics.tex

\renewcommand{\FIGREP}{src/serial/figures}

\section{Compilation}
\label{sec:compilation}
\intersec{deneb}

\subsection{Basics of compilation}
\label{sec:basics}

\begin{frame}
  \frametitle{Compilation}
  \framesubtitle{0100101110101001010...}
  \begin{itemize}
    \item A computer only understands ON and OFF states (1 and 0)
    \item It would be very inconvenient for us to code in binary
    \item We therefore use different levels of abstraction (languages), e.g. C, C++, Fortran
    \item We need a translator!
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Compilation}
  \framesubtitle{The four compilation steps}
  \begin{itemize}
    \item Translation  is made by a compiler in 4 steps
    \begin{description}
      \item[Preprocessing] Format source code to make it ready for compilation (remove comments, execute preprocessing directives such as \cmd{\#include}, etc.)
      \item[Compiling] Translate the source code (C, C++, Fortran, etc) into assembly, a very basic CPU-dependent language
      \item[Assembly] Translate the assembly into machine code and store it in object files
      \item[Linking] Link all the object files into one executable
    \end{description}
    \item In practice, the first three steps are combined together and simply
          called ``compiling''
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{Compilation}
  \framesubtitle{The four compilation steps (visually)}
  \hspace{6cm}
  \begin{minipage}{0.5\textwidth}
    \begin{itemize}
      \item<5> Note that in reality, everything is done transparently
            \begin{bashcode}
              $> gcc -c file_1.c
              $> gcc -c file_2.c
              $> gcc file_1.o file_2.o -lexample -o exec
            \end{bashcode}%$
    \end{itemize}
  \end{minipage}

  \onslide<1>\addimage[width=12cm]{\FIGREP/compilation_steps_0.pdf}{2cm}{1cm}
  \onslide<2>\addimage[width=12cm]{\FIGREP/compilation_steps_1.pdf}{2cm}{1cm}
  \onslide<3>\addimage[width=12cm]{\FIGREP/compilation_steps_2.pdf}{2cm}{1cm}
  \onslide<4>\addimage[width=12cm]{\FIGREP/compilation_steps_3.pdf}{2cm}{1cm}
  \onslide<5>\addimage[width=12cm]{\FIGREP/compilation_steps_4.pdf}{2cm}{1cm}
\end{frame}

\begin{frame}
  \frametitle{Compilation}
  \framesubtitle{Let's start from the beginning}

  \begin{itemize}
    \item There are many compilers for different languages (here C, C++, and
    Fortran)
    \begin{itemize}
      \item GNU (\cmd{gcc}, \cmd{g++}, \cmd{gfortran})
      \item Clang/LLVM
      \item Intel (\cmd{icc}, \cmd{icpc}, \cmd{ifort}) and Intel OneAPI (\cmd{icx}, \cmd{icpx}, \cmd{ifx})
      \item IBM (\cmd{xlc}, \cmd{xlc++}, \cmd{xlf})
      \item etc
    \end{itemize}
    \item Each vendor has specific options and usage
  \end{itemize}
  \vfill
  \textbf{In the following, we will use GNU compilers, but everything can be
    adapted to other vendors}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Compiling a single source file}
  \framesubtitle{}

  \begin{itemize}
    \item In general, the command to compile a single source file is
    \begin{bashcode}
      <compiler> <compiling options> <source file> <linking options>
    \end{bashcode}
    \item For example
    \begin{bashcode}
      gcc -o my_exe code.c
    \end{bashcode}
    I want to compile the file \cmd{code.c} using the GNU C compiler and with
    option \cmd{-o my_exe} (rename the executable to \cmd{my_exe} instead
    of the default \cmd{a.out})
    \item We will look at the linking later in the course
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Compilation}
  \framesubtitle{Compilation options}
  Many compilation options can be used to manipulate, optimize or debug such as:
  \begin{itemize}
    \item "Manipulation": \cmd{-o}, \cmd{-c}, etc.
    \item "Optimization": \cmd{-On}, \cmd{-fastmath}, etc.
    \item "Debug": \cmd{-g}, \cmd{-Wall}, \cmd{-fcheck=all} etc.
    \item Option summary for GNU: \url{https://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html\#Invoking-GCC}
  \end{itemize}
\end{frame}

\begin{frame}[exercise]
  \frametitle{Compilation}
  \framesubtitle{Exercise simpleCompilation}
  The following code performs a simple saxpy, \ie{} $\vec{z} = \alpha \vec{x} + \vec{y}$
  \begin{itemize}
    \item Load gcc module with \cmd{module load gcc}
    \item Go to the directory \cmd{simpleCompilation}
    \item Compile the code \cmd{saxpy.F90} with no options and execute it. What
          happens?
    \item Make the code compile with \cmd{-O0 -g -Werror -Wall -Wextra
          -fcheck=all} and run it. What happens?
    \item Try different compilation optimization options (\cmd{-O0} to
          \cmd{-O3}) and compare the timing displayed from the corresponding
          runs
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Discussion}
  \framesubtitle{Exercise simpleCompilation}
  \begin{itemize}
    \item Compilers greatly help you to debug and optimize your code. Use them
    properly!
    \item We advise you to begin by disabling optimizations and activating warnings
    \begin{itemize}
      \item \cmd{-O0}: turn off most optimizations (default with GCC)
      \item \cmd{-Wall -Wextra}: activate most of the warnings
      \item \cmd{-Werror}: treat warnings as errors
      \item \cmd{-fcheck=all}: enable all run-time checks
    \end{itemize}
    \item Some errors can be easily avoided using the proper flags!
    \item Once Everything is correct, remove those options and try optimization
    flags
    \item \cmd{-O3} is not necessarily faster than \cmd{-O2}. Test it!
  \end{itemize}
\end{frame}

\begin{frame}[exercise]
  \frametitle{Compilation}
  \framesubtitle{Exercise compilationWith2Files}
  \begin{itemize}
    \item Go to the directory \cmd{compilationWith2Files}
    \item The previous code has been split into two files: \cmd{main.F90} and \cmd{saxpy.F90}
    \item Compile and execute the code. Hint, use the \cmd{-c} option.
    \item Check the results compared to the \cmd{simpleCompilation} exercise
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{Discussion}
  \framesubtitle{Exercise compilationWith2Files}
  \begin{itemize}
    \item With multiple source files, one must compile them separately and link
          them together at the end
    \item The \cmd{-c} option compiles, but does not link
    \item Remember to pass all the options to each file!
    \item This quickly becomes cumbersome as the number of files increases. We
    will see possible solutions later.
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{Preprocessor}
  \framesubtitle{}

  \begin{itemize}
    \item The C preprocessor, \cmd{cpp}, is a tool that modifies the source code prior to
    compilation
    \item It treats all the lines starting with \cmd{\#}, \eg{}
    \begin{bashcode}
      #include <iostream>
    \end{bashcode}
    \item You can define variables, macros, etc.
    \pause
    \item It can also take variables from the compilation command, \eg{}
    \begin{bashcode}
      $> gcc -DMYVAR code.c
    \end{bashcode}%$
    The option \cmd{-D} will pass variable \cmd{MYVAR} to the preprocessor
    \pause
    \item By default, \cmd{gcc} calls the preprocessor before compiling
    \item For \cmd{gfortran}, you have to ask it:
    \begin{itemize}
      \item Use capital file extensions, \eg{} \cmd{.F90}, \cmd{.F}, \cmd{.F03}, etc.
      \item Add the \cmd{-cpp} option to the compiler
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile, exercise]
  \frametitle{Preprocessor}
  \framesubtitle{}

\end{frame}

\begin{frame}
  \frametitle{Libraries}
  \framesubtitle{}
  \begin{itemize}
    \item Libraries are a collection of pre-compiled pieces of code that can be
          reused in other programs
    \item One almost always uses libraries in our programs, \eg{}
    \cmd{iostream}, \cmd{vector} in C++, \cmd{stdio} in C
    \pause
    \item In general, a library consists of a header file, \cmd{cmath.h}, and a
    binary object, \cmd{libm.so}
    \item At compile time (pre-processing) you provide the headers
    \item At linking time you provide the object files
  \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Libraries}
  \framesubtitle{A simple example}

  \begin{itemize}
    \item Someone wrote an awesome math library called \cmd{mymath} providing
    the \cmd{squared} function
    \item Let's us it!
  \end{itemize}

  \cxxfile[title={libraryExample/main\_bad.c}]{examples/libraryExample/main_bad.c}
  \pause
  \begin{bashcode}
    $> gcc -Werror -Wall main_bad.c -o main
       main_bad.c: In function ‘main’:
       main_bad.c:4:39: error: implicit declaration of function ‘squared’
       4 |   printf("The square of 2.0 is %f\n", squared(2.0));
     \end{bashcode}%$
     \begin{itemize}
       \item Our program doesn't know about this \cmd{squared} function
     \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Libraries}
  \framesubtitle{A (better) simple example}

  \begin{itemize}
    \item Let's add the header file to declare the \cmd{squared} function
  \end{itemize}

  \cxxfile[title={libraryExample/main\_correct.c}]{examples/libraryExample/main_correct.c}
  \pause
  \begin{bashcode}
    $> gcc -Werror -Wall main_correct.c -o main
       main_correct.c:2:10: fatal error: mymath.h: No such file or directory
       2 | #include "mymath.h"
       |          ^~~~~~~~~~
       compilation terminated.
     \end{bashcode}%$
     \begin{itemize}
       \item We need to tell the compiler where to find this header file
     \end{itemize}
\end{frame}

\begin{frame}[fragile]
  \frametitle{Libraries}
  \framesubtitle{Include header files}
  \begin{itemize}
    \item When you have an \cmd{#include} or \cmd{use} (f90) in your code, the
    compiler must know where to look for them
    \item It has some default paths it will always look in, \eg{} system paths
    and current path, but you can supply your own with the \cmd{-I} option
  \end{itemize}

  \textbf{Example}:
  \begin{itemize}
    \item Let's assume the \cmd{mylibrary.h} header is located in
    \cmd{~/mycode/includes/mylibrary.h}
    \item The source code that uses it is located in
    \cmd{~/mycode/src/source.cpp}
    \item Here you have two options:
    \begin{enumerate}
      \item You put the library path in the source file, \ie{} \cmd{#include
        "../includes/mylibrary.h"}
      \item You simply use \cmd{#include "mylibrary.h"} and tell the compiler (pre-processor)
      where to look for it
      \begin{bashcode}
        gcc -I~/mycode/includes source.cpp
      \end{bashcode}
    \end{enumerate}
  \end{itemize}
\end{frame}

\begin{frame}[fragile,t]
  \frametitle{Libraries}
  \framesubtitle{A (almost there) simple example}

  \begin{itemize}
    \item Let's add the correct option to the compiler
  \end{itemize}

  \begin{bashcode}
    $> gcc -Imylibrary -Werror -Wall main_correct.c -o main
       /usr/bin/ld: /tmp/ccBYOiab.o: in function `main':
       main_correct.c:(.text+0x11): undefined reference to `squared'
       collect2: error: ld returned 1 exit status
     \end{bashcode}%$
     \begin{itemize}
       \item Now we are missing the actual definition of the function
       \item Two possibilities, the bad and the correct
     \end{itemize}
  \begin{bashcode}
    $> gcc -Imylibrary main_correct.c -o main mylibrary/libmymath.so  # the bad
     \end{bashcode}%$
     \begin{bashcode}
    $> gcc -Imylibrary main_correct.c -o main -Lmylibrary -lmymath    # the correct
     \end{bashcode}%$
   \end{frame}

\begin{frame}[fragile,t]
  \frametitle{Libraries}
  \framesubtitle{Linking options}

  \begin{itemize}
    \item To link a library to your executable you need to tell the linker
    \begin{itemize}
      \item Where is the library: \cmd{-L<PATH/TO/LIBRARY>}
      \item Which library you want to use: \cmd{-l<LIB_NAME>}
    \end{itemize}
    \item The linking options should be put at the end (after the files against
    which they will be linked)
    \begin{bashcode}
      $> gcc -Imylibrary main_correct.c -o main -Lmylibrary -lmymath    # OK
    \end{bashcode}%$
    \begin{bashcode}
      $> gcc -Lmylibrary -lmymath -Imylibrary main_correct.c -o main    # NOT OK
    \end{bashcode}%$
    \item In some cases, the order with which the library names appear matters.
    Read the linker documentation!
  \end{itemize}
  \vfill
  \pause
  \textbf{Summary:}
      \begin{bashcode}
      $> gcc -I<PATH/TO/HEADERS> code.c -L<PATH/TO/LIBRARY> -l<LIBRARY_NAME>
    \end{bashcode}%$
  \end{frame}

\begin{frame}[fragile,b]
  \frametitle{Static and shared libraries}
  \framesubtitle{}

  \begin{minipage}{0.49\linewidth}
    \begin{itemize}
      \item Name is \cmd{libXYZ.a}
      \item Lib linked at compile time
      \item If the library changes, must recompile the program
      \item Executable is larger
    \end{itemize}
  \end{minipage}
  \hfill
  \begin{minipage}{0.49\linewidth}
    \begin{itemize}
      \item Name is \cmd{libXYZ.so}
      \item They are ``linked'' to the executable
      \item A table is created at compile time and libs are loaded at runtime
      \item Libs can be updated independently
      \item Smaller executable
    \end{itemize}
  \end{minipage}

  \addimage[width=10cm]{\FIGREP/library_static_shared}{3.0cm}{3.3cm}
  \end{frame}

\begin{frame}[t,fragile]
  \frametitle{Shared libraries}
  \framesubtitle{Loading the libraries}

  The following applies for shared libraries, but doesn't impact static
  libraries

  \begin{itemize}
    \item Shared libraries are not included in the executable, but only linked
    and loaded at runtime
    \item You can check them with \cmd{ldd}
    \begin{bashcode}
      $> ldd main
         linux-vdso.so.1 (0x00007ffc716df000)
         libmymath.so => not found
         libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f80e0657000)
         /lib64/ld-linux-x86-64.so.2 (0x00007f80e0846000)
    \end{bashcode}%$
    \item To tell the OS where is the library to load, we use the variable
    \cmd{LD_LIBRARY_PATH}
    \item For example, prepend your path
      \begin{bashcode}
        $> export LD_LIBRARY_PATH=/path/to/the/library:$LD_LIBRARY_PATH
      \end{bashcode}
  \end{itemize}
\end{frame}

\begin{frame}[exercise]
  \frametitle{Compilation}
  \framesubtitle{Exercise compilationWithLibrary}
  \begin{itemize}
    \item Go to the directory \cmd{compilationWithLibrary}
    \item Compile the code \cmd{main.F90} with shared library \cmd{libsaxpy.so}
    \item Execute the code
    \item Check the results according to the solution of exercise \cmd{simpleCompilation}
  \end{itemize}
\end{frame}

\begin{frame}[exercise]
  \frametitle{Compilation}
  \framesubtitle{BONUS: Exercise compilationWithMixLibrary}
  \begin{itemize}
    \item To mix static and shared libraries use: \cmd{-Wl,-Bstatic static_library -Wl,-Bdynamic shared_library}
    \item Go to the directory \cmd{compilationWithMixLibrary}
    \item Compile the code \cmd{main.c} with static library \cmd{libmax.a} and shared library \cmd{libsaxpy.so}
    \item Execute the code
    \item Check the results according to the solution of exercise \cmd{simpleCompilation}
  \end{itemize}
\end{frame}

\begin{frame}
  \frametitle{A few notable libraries}
  \framesubtitle{}

  Don't reinvent the wheel, use a library!
  \vfill
  \begin{itemize}
    \item Basic Linear Algebra Subprograms (BLAS)
    \begin{itemize}
      \item Provides routines to perform basic linear algebra operations (vector
      addition, dot product, etc.)
      \item OpenBLAS, Intel MKL, cuBLAS, rocBLAS, etc.
    \end{itemize}
    \item Linear Algebra Package (LAPACK)
    \begin{itemize}
      \item Provides routines for solving systems of linear equations
      \item OpenBLAS, Intel MKL, Netlib LAPACK and ScaLAPACK, etc.
    \end{itemize}
    \item Other solvers (direct and iterative)
    \begin{itemize}
      \item MUMPS, SuperLU, PETSc, PARDISO, etc.
    \end{itemize}
    \item Fourier transforms
    \begin{itemize}
      \item FFTW, cuFFT, intel MKL, etc.
    \end{itemize}
    \item And many others!
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{BONUS: Making your own library}
  \framesubtitle{}
  \begin{itemize}
    \item Creating static library \cmd{libmylib.a} from different object files \cmd{prog*.o}
    \begin{consoleoutput}
      $ gcc -c prog1.c prog2.c prog3.c ... progn.c
      $ ar -rv libmylib.a prog1.o prog2.o prog3.o ... progn.o
    \end{consoleoutput}
    \item Creating shared library \cmd{libmylib.so} from different object files \cmd{prog*.o}
    \begin{consoleoutput}
      $ gcc -fPIC -c prog1.c prog2.c prog3.c ... progn.c
      $ gcc -o libmylib.so -shared prog1.o prog2.o prog3.o ... progn.o
    \end{consoleoutput}
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{Build automation tool}
  \framesubtitle{GNU Make}
  \begin{itemize}
    \item Compiling one file by hand is fine, but multiple files become quickly
    cumbersome
    \item Tools have been developed to automatize such process
    \item Here, we will only briefly introduce GNU Make and CMake
  \end{itemize}

  \vfill

  \textbf{GNU Make aka Makefile:}
  \begin{itemize}
    \item The command line \cmd{make} executes shell commands containing in a
    makefile (list of rules)
    \item Rules contain three components:
    \begin{bashcode}
      target: prerequisites
      ->|     recipe
      ->|     ...
    \end{bashcode}
    \item Make keeps track of timestamps: it updates only targets that are
    required
    \item Make can be used for a lot of other things than compiling code!
    \item Just type \cmd{make} in the folder containing the \cmd{Makefile} file
  \end{itemize}
\end{frame}
\note{
   \begin{itemize}  
     \item If you have a large program with many source and/or header files, when you change a file on which others depend, you must recompile all the dependent files. Without a makefile, this is an extremely time-consuming task.   
     \item The makefile contains a list of rules. These rules tell the system what commands you want to be executed. Most times, these rules are commands to compile(or recompile) a series of files. The rules, which must begin in column 1, are in two parts. The first line is called a dependency line and the subsequent line(s) are called actions or commands. The action line(s) must be indented with a tab.  
     \item CMake is used to control the software compilation process using simple platform and compiler independent configuration files, and generate native makefiles and workspaces that can be used in the compiler environment of your choice
   \end{itemize}
 }

 \begin{frame}[t,fragile]
  \frametitle{Makefile}
  \framesubtitle{First naive example}
  \bashfile[title={examples/compilationWithMakefile/Makefile}]{examples/compilationWithMakefile/Makefile}
\end{frame}
\note{
   \begin{itemize}  
     \item First, we can use in the Makefile some variables to assign commands
     \item On the left you have the so-called target: all, ... and following the target, the dependencies %my_exec, etc ... and following the target: the dependencies
     \item The recipes are introduced below the definition target:dependencies
     \item A target might be a binary file that depends on dependencies (source files). On the other hand, a dependency can also be a target that depends on other dependencies:
   \end{itemize}
}

\begin{frame}[t,fragile]
  \frametitle{Makefile}
  \framesubtitle{More refined example}
  \bashfile[title={examples/compilationWithMakefileAdvanced/Makefile}]{examples/compilationWithMakefileAdvanced/Makefile}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{Build automation tool}
  \framesubtitle{GNU CMake}

  \textbf{GNU CMake:}
  \begin{itemize}
    \item CMake is an abstraction level above Make
    \item It automatically detects compilers and libraries and generate a
    Makefile (or other tools, \eg{} Ninja)
    \item Suggested workflow:
    \begin{itemize}
      \item Create a build directory and go inside of it
      \begin{bashcode}
        mkdir build
        cd build
      \end{bashcode}
      \item Configure the project
      \begin{bashcode}
        cmake ..
      \end{bashcode}
      \item Compile it
      \begin{bashcode}
        make
      \end{bashcode}
    \end{itemize}
    \item When working, CMake is a very nice tool! Otherwise, it requires some
    knowledge to debug.
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{CMake}
  \framesubtitle{Configuration output}

   \begin{consoleoutput}
   $ cmake ..
     -- The C compiler identification is GNU 8.4.0
     -- Check for working C compiler: /ssoft/.../gcc-4.8.5/gcc-8.4.0/bin/gcc
     -- Check for working C compiler: /ssoft/.../gcc-4.8.5/gcc-8.4.0/bin/gcc -- works
     -- Detecting C compiler ABI info
     -- Detecting C compiler ABI info - done
     -- Detecting C compile features
     -- Detecting C compile features - done
     -- Configuring done
     -- Generating done
     -- Build files have been written to: compilationWithCmake/build
   \end{consoleoutput}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{CMake example}
  \framesubtitle{}
  \bashfile[title={examples/compilationWithCmake/CMakeLists.txt}]{examples/compilationWithCmake/CMakeLists.txt}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{Compilation}
  \framesubtitle{CMake example}
  \bashfile[title={examples/compilationWithCmakeAdvanced/CMakeLists.txt},
  minted options app={firstline=5,lastline=29}]{examples/compilationWithCmakeAdvanced/CMakeLists.txt}  
\end{frame}

\begin{frame}[exercise]
  \frametitle{Compilation}
  \framesubtitle{Exercise compilationWithCmakeAdvanced}
  \begin{itemize}
    \item Go to the directory \cmd{compilationWithCmakeAdvanced}
    \item Create a directory \cmd{build_gcc} and \cmd{build_intel}
    \item Compile the code, first with \cmd{gcc} compiler in \cmd{build_gcc}, then in \cmd{build_intel} with \cmd{intel} compiler
    \item Use different compilation options modifying the \cmd{CMakeLists.txt} file or using the \cmd{cmake} command line with \cmd{-DCMAKE_C_FLAGS="..."} option
    \item Use \cmd{ifdef MY_MACRO} introduced in \cmd{main.c} using the \cmd{cmake} command line with \cmd{-DSAXPY_USE_MY_MACRO="..."} option or ccmake 
    \item Check the results according to the solution of exercise \cmd{simpleCompilation}
  \end{itemize}
\end{frame}

\begin{frame}[t,fragile]
  \frametitle{Cheat sheet}
  \framesubtitle{}

\end{frame}
%%% Local Variables:
%%% mode: latex
%%% TeX-master: "../../SCM_slides"
%%% End: