glibc/sysdeps/aarch64/fpu/math_private.h

/* Private floating point rounding and exceptions handling.  AArch64 version.
   Copyright (C) 2014-2018 Free Software Foundation, Inc.
   This file is part of the GNU C Library.

   The GNU C Library is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; either
   version 2.1 of the License, or (at your option) any later version.

   The GNU C Library is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should have received a copy of the GNU Lesser General Public
   License along with the GNU C Library; if not, see
   <http://www.gnu.org/licenses/>.  */

#ifndef AARCH64_MATH_PRIVATE_H
#define AARCH64_MATH_PRIVATE_H 1

#include <fenv.h>
#include <fpu_control.h>

#define math_opt_barrier(x) \
({ __typeof (x) __x = (x); __asm ("" : "+w" (__x)); __x; })
#define math_force_eval(x) \
({ __typeof (x) __x = (x); __asm __volatile__ ("" : : "w" (__x)); })

extern __always_inline double
__ieee754_sqrt (double d)
{
  double res;
  asm __volatile__ ("fsqrt   %d0, %d1" : "=w" (res) : "w" (d));
  return res;
}

extern __always_inline float
__ieee754_sqrtf (float s)
{
  float res;
  asm __volatile__ ("fsqrt   %s0, %s1" : "=w" (res) : "w" (s));
  return res;
}

static __always_inline void
libc_feholdexcept_aarch64 (fenv_t *envp)
{
  fpu_control_t fpcr;
  fpu_control_t new_fpcr;
  fpu_fpsr_t fpsr;
  fpu_fpsr_t new_fpsr;

  _FPU_GETCW (fpcr);
  _FPU_GETFPSR (fpsr);
  envp->__fpcr = fpcr;
  envp->__fpsr = fpsr;

  /* Clear exception flags and set all exceptions to non-stop.  */
  new_fpcr = fpcr & ~(FE_ALL_EXCEPT << FE_EXCEPT_SHIFT);
  new_fpsr = fpsr & ~FE_ALL_EXCEPT;

  if (__glibc_unlikely (new_fpcr != fpcr))
    _FPU_SETCW (new_fpcr);

  if (new_fpsr != fpsr)
    _FPU_SETFPSR (new_fpsr);
}

#define libc_feholdexcept  libc_feholdexcept_aarch64
#define libc_feholdexceptf libc_feholdexcept_aarch64
#define libc_feholdexceptl libc_feholdexcept_aarch64

static __always_inline void
libc_fesetround_aarch64 (int round)
{
  fpu_control_t fpcr;

  _FPU_GETCW (fpcr);

  /* Check whether rounding modes are different.  */
  round = (fpcr ^ round) & _FPU_FPCR_RM_MASK;

  /* Set new rounding mode if different.  */
  if (__glibc_unlikely (round != 0))
    _FPU_SETCW (fpcr ^ round);
}

#define libc_fesetround  libc_fesetround_aarch64
#define libc_fesetroundf libc_fesetround_aarch64
#define libc_fesetroundl libc_fesetround_aarch64

static __always_inline void
libc_feholdexcept_setround_aarch64 (fenv_t *envp, int round)
{
  fpu_control_t fpcr;
  fpu_control_t new_fpcr;
  fpu_fpsr_t fpsr;
  fpu_fpsr_t new_fpsr;

  _FPU_GETCW (fpcr);
  _FPU_GETFPSR (fpsr);
  envp->__fpcr = fpcr;
  envp->__fpsr = fpsr;

  /* Clear exception flags, set all exceptions to non-stop,
     and set new rounding mode.  */
  new_fpcr = fpcr & ~((FE_ALL_EXCEPT << FE_EXCEPT_SHIFT) | _FPU_FPCR_RM_MASK);
  new_fpcr |= round;
  new_fpsr = fpsr & ~FE_ALL_EXCEPT;

  if (__glibc_unlikely (new_fpcr != fpcr))
    _FPU_SETCW (new_fpcr);

  if (new_fpsr != fpsr)
    _FPU_SETFPSR (new_fpsr);
}

#define libc_feholdexcept_setround  libc_feholdexcept_setround_aarch64
#define libc_feholdexcept_setroundf libc_feholdexcept_setround_aarch64
#define libc_feholdexcept_setroundl libc_feholdexcept_setround_aarch64

static __always_inline int
libc_fetestexcept_aarch64 (int ex)
{
  fpu_fpsr_t fpsr;

  _FPU_GETFPSR (fpsr);
  return fpsr & ex & FE_ALL_EXCEPT;
}

#define libc_fetestexcept  libc_fetestexcept_aarch64
#define libc_fetestexceptf libc_fetestexcept_aarch64
#define libc_fetestexceptl libc_fetestexcept_aarch64

static __always_inline void
libc_fesetenv_aarch64 (const fenv_t *envp)
{
  fpu_control_t fpcr;
  fpu_control_t new_fpcr;

  _FPU_GETCW (fpcr);
  new_fpcr = envp->__fpcr;

  if (__glibc_unlikely (fpcr != new_fpcr))
    _FPU_SETCW (new_fpcr);

  _FPU_SETFPSR (envp->__fpsr);
}

#define libc_fesetenv  libc_fesetenv_aarch64
#define libc_fesetenvf libc_fesetenv_aarch64
#define libc_fesetenvl libc_fesetenv_aarch64
#define libc_feresetround_noex  libc_fesetenv_aarch64
#define libc_feresetround_noexf libc_fesetenv_aarch64
#define libc_feresetround_noexl libc_fesetenv_aarch64

static __always_inline int
libc_feupdateenv_test_aarch64 (const fenv_t *envp, int ex)
{
  fpu_control_t fpcr;
  fpu_control_t new_fpcr;
  fpu_fpsr_t fpsr;
  fpu_fpsr_t new_fpsr;
  int excepts;

  _FPU_GETCW (fpcr);
  _FPU_GETFPSR (fpsr);

  /* Merge current exception flags with the saved fenv.  */
  excepts = fpsr & FE_ALL_EXCEPT;
  new_fpcr = envp->__fpcr;
  new_fpsr = envp->__fpsr | excepts;

  if (__glibc_unlikely (fpcr != new_fpcr))
    _FPU_SETCW (new_fpcr);

  if (fpsr != new_fpsr)
    _FPU_SETFPSR (new_fpsr);

  /* Raise the exceptions if enabled in the new FP state.  */
  if (__glibc_unlikely (excepts & (new_fpcr >> FE_EXCEPT_SHIFT)))
    __feraiseexcept (excepts);

  return excepts & ex;
}

#define libc_feupdateenv_test  libc_feupdateenv_test_aarch64
#define libc_feupdateenv_testf libc_feupdateenv_test_aarch64
#define libc_feupdateenv_testl libc_feupdateenv_test_aarch64

static __always_inline void
libc_feupdateenv_aarch64 (const fenv_t *envp)
{
  libc_feupdateenv_test_aarch64 (envp, 0);
}

#define libc_feupdateenv  libc_feupdateenv_aarch64
#define libc_feupdateenvf libc_feupdateenv_aarch64
#define libc_feupdateenvl libc_feupdateenv_aarch64

static __always_inline void
libc_feholdsetround_aarch64 (fenv_t *envp, int round)
{
  fpu_control_t fpcr;
  fpu_fpsr_t fpsr;

  _FPU_GETCW (fpcr);
  _FPU_GETFPSR (fpsr);
  envp->__fpcr = fpcr;
  envp->__fpsr = fpsr;

  /* Check whether rounding modes are different.  */
  round = (fpcr ^ round) & _FPU_FPCR_RM_MASK;

  /* Set new rounding mode if different.  */
  if (__glibc_unlikely (round != 0))
    _FPU_SETCW (fpcr ^ round);
}

#define libc_feholdsetround  libc_feholdsetround_aarch64
#define libc_feholdsetroundf libc_feholdsetround_aarch64
#define libc_feholdsetroundl libc_feholdsetround_aarch64

static __always_inline void
libc_feresetround_aarch64 (fenv_t *envp)
{
  fpu_control_t fpcr;
  int round;

  _FPU_GETCW (fpcr);

  /* Check whether rounding modes are different.  */
  round = (envp->__fpcr ^ fpcr) & _FPU_FPCR_RM_MASK;

  /* Restore the rounding mode if it was changed.  */
  if (__glibc_unlikely (round != 0))
    _FPU_SETCW (fpcr ^ round);
}

#define libc_feresetround  libc_feresetround_aarch64
#define libc_feresetroundf libc_feresetround_aarch64
#define libc_feresetroundl libc_feresetround_aarch64

/* We have support for rounding mode context.  */
#define HAVE_RM_CTX 1

static __always_inline void
libc_feholdsetround_aarch64_ctx (struct rm_ctx *ctx, int r)
{
  fpu_control_t fpcr;
  int round;

  _FPU_GETCW (fpcr);
  ctx->env.__fpcr = fpcr;

  /* Check whether rounding modes are different.  */
  round = (fpcr ^ r) & _FPU_FPCR_RM_MASK;
  ctx->updated_status = round != 0;

  /* Set the rounding mode if changed.  */
  if (__glibc_unlikely (round != 0))
    _FPU_SETCW (fpcr ^ round);
}

#define libc_feholdsetround_ctx		libc_feholdsetround_aarch64_ctx
#define libc_feholdsetroundf_ctx	libc_feholdsetround_aarch64_ctx
#define libc_feholdsetroundl_ctx	libc_feholdsetround_aarch64_ctx

static __always_inline void
libc_feresetround_aarch64_ctx (struct rm_ctx *ctx)
{
  /* Restore the rounding mode if updated.  */
  if (__glibc_unlikely (ctx->updated_status))
    _FPU_SETCW (ctx->env.__fpcr);
}

#define libc_feresetround_ctx		libc_feresetround_aarch64_ctx
#define libc_feresetroundf_ctx		libc_feresetround_aarch64_ctx
#define libc_feresetroundl_ctx		libc_feresetround_aarch64_ctx

static __always_inline void
libc_feholdsetround_noex_aarch64_ctx (struct rm_ctx *ctx, int r)
{
  fpu_control_t fpcr;
  fpu_fpsr_t fpsr;
  int round;

  _FPU_GETCW (fpcr);
  _FPU_GETFPSR (fpsr);
  ctx->env.__fpcr = fpcr;
  ctx->env.__fpsr = fpsr;

  /* Check whether rounding modes are different.  */
  round = (fpcr ^ r) & _FPU_FPCR_RM_MASK;
  ctx->updated_status = round != 0;

  /* Set the rounding mode if changed.  */
  if (__glibc_unlikely (round != 0))
    _FPU_SETCW (fpcr ^ round);
}

#define libc_feholdsetround_noex_ctx	libc_feholdsetround_noex_aarch64_ctx
#define libc_feholdsetround_noexf_ctx	libc_feholdsetround_noex_aarch64_ctx
#define libc_feholdsetround_noexl_ctx	libc_feholdsetround_noex_aarch64_ctx

static __always_inline void
libc_feresetround_noex_aarch64_ctx (struct rm_ctx *ctx)
{
  /* Restore the rounding mode if updated.  */
  if (__glibc_unlikely (ctx->updated_status))
    _FPU_SETCW (ctx->env.__fpcr);

  /* Write new FPSR to restore exception flags.  */
  _FPU_SETFPSR (ctx->env.__fpsr);
}

#define libc_feresetround_noex_ctx	libc_feresetround_noex_aarch64_ctx
#define libc_feresetround_noexf_ctx	libc_feresetround_noex_aarch64_ctx
#define libc_feresetround_noexl_ctx	libc_feresetround_noex_aarch64_ctx

/* Hack: only include the large arm_neon.h when needed.  */
#ifdef _MATH_CONFIG_H
# include <arm_neon.h>

/* ACLE intrinsics for frintn and fcvtns instructions.  */
# define TOINT_INTRINSICS 1

static inline double_t
roundtoint (double_t x)
{
  return vget_lane_f64 (vrndn_f64 (vld1_f64 (&x)), 0);
}

static inline uint64_t
converttoint (double_t x)
{
  return vcvtnd_s64_f64 (x);
}
#endif

#include_next <math_private.h>

#endif