%%% Everything to right from '%' is a comment; does not show in the final pdf file and can be deleted.
%%% DO NOT EDIT the following section enclosed by *****
%%% ***************************************************
\documentclass[twocolumn,amsmath,amssymb,10pt,superscriptaddress,a4paper,letterpaper,fleqn,floatfix]{revtex4-1}
\usepackage{amssymb}
\usepackage{epsfig}
\usepackage{graphicx}
\usepackage{dcolumn}
\usepackage{array}
\usepackage{bm}
\usepackage{fancyheadings}
\usepackage{longtable}
\usepackage{multirow}
\usepackage{float}
\pagestyle{fancy}
\usepackage{afterpage}
\usepackage{color}
\usepackage[table]{xcolor}


\bibliographystyle{ieeetr.bst}
\setlongtables
\usepackage[breaklinks=true,linkbordercolor={1 1 1}]{hyperref}

\parskip 0pc
\textwidth=18cm
\columnwidth=9cm

\setcounter{secnumdepth}{5}

%\def\etal{et al.\,}
\def\etal{\textit{et al.}}
\def\etc{\textit{etc.}}
\def\eg{\textit{e.g.}}
\def\ie{\textit{i.e.}}
\newcommand{\mub}{$\bar{\mu}$}
\newcommand{\nub}{$\bar{\nu}$}
\newcommand{\kef}{$k_{\rm eff}$}
\newcommand{\cgs}{\cellcolor[gray]{0.8}}

\begin{document}
\setcounter{page}{1}

%%% **********************************************************************

\title{
%% Please do not remove the line below
\qquad \\ \qquad \\ \qquad \\  \qquad \\  \qquad \\ \qquad \\
Methods and Issues for the Combined Use of Integral Experiments and Covariance Data: Results of a NEA International Collaborative Study}

\author{M.~Salvatores}
\email[Corresponding author: ]{massimo.salvatores@cea.fr}
\affiliation{CEA, Nuclear Energy Division, Cadarache, Saint-Paul-lez-Durance, France}
\affiliation{Idaho National Laboratory (INL), Idaho Falls, ID, USA}
\author{G.~Palmiotti}
\affiliation{Idaho National Laboratory (INL), Idaho Falls, ID, USA}
\author{G.~Aliberti}
\affiliation{Argonne National Laboratory (ANL), Argonne, IL, USA}
\author{P.~Archier}
\affiliation{CEA, Nuclear Energy Division, Cadarache, Saint-Paul-lez-Durance, France}
\author{C.~De Saint Jean}
\affiliation{CEA, Nuclear Energy Division, Cadarache, Saint-Paul-lez-Durance, France}
\author{E.~Dupont}
\affiliation{OECD, Nuclear Energy Agency (NEA), Issy-les-Moulineaux, France}
\author{M.~Herman}
\affiliation{Brookhaven National Laboratory (BNL), Upton, NY, USA}
\author{M.~Ishikawa}
\affiliation{Japan Atomic Energy Agency (JAEA), Tokai-mura, Ibaraki, Japan}
\author{T.~Ivanova}
\affiliation{Institut de Radioprotection et de Surete Nucleaire (IRSN), Fontenay-aux-Roses, France}
\author{E.~Ivanov}
\affiliation{Institut de Radioprotection et de Surete Nucleaire (IRSN), Fontenay-aux-Roses, France}
\author{S.-J.~Kim}
\affiliation{Korea Atomic Energy Research Institute (KAERI), Daejeon, South Korea}
\author{I.~Kodeli}
\affiliation{Jozef Stefan Institute (JSI), Ljubljana, Slovenia}
\author{G.~Manturov}
\affiliation{Institute of Physics and Power Engineering (IPPE), Obninsk, Russia}
\author{R.~McKnight}
\affiliation{Argonne National Laboratory (ANL), Argonne, IL, USA}
\author{S.~Pelloni}
\affiliation{Paul Scherrer Institut (PSI), Villigen, Switzerland}
\author{C.~Perfetti}
\affiliation{Oak Ridge National Laboratory (ORNL), Oak Ridge, TN, USA}
\author{A.~Plompen}
\affiliation{Institute for Reference Materials and Measurements (IRMM), Geel, Belgium}
\author{B.T.~Rearden}
\affiliation{Oak Ridge National Laboratory (ORNL), Oak Ridge, TN, USA}
\author{D.~Rochman}
\affiliation{Nuclear Research and Consultancy Group (NRG), Petten, The Netherlands}
\author{K.~Sugino}
\affiliation{Japan Atomic Energy Agency (JAEA), Tokai-mura, Ibaraki, Japan}
\author{A.~Trkov}
\affiliation{Jozef Stefan Institute (JSI), Ljubljana, Slovenia}
\author{W.~Wang}
\affiliation{China Institute of Atomic Energy (CIAE), China Nuclear Data Center, Beijing, China}
\author{H.~Wu}
\affiliation{China Institute of Atomic Energy (CIAE), China Nuclear Data Center, Beijing, China}
\author{W.-S.~Yang}
\affiliation{Argonne National Laboratory (ANL), Argonne, IL, USA}
\affiliation{Purdue University, West Lafayette, IN, USA}

\date{\today} 
%\received{8 March 2013; revised received XX June 2013; accepted XX September 2013}

\begin{abstract}
The Working Party on International Nuclear Data Evaluation Cooperation (WPEC) of the OECD NEA Nuclear Science
Committee established in 2009 a Subgroup (called ``Subgroup 33'') on ``Methods and issues for the combined use of
integral experiments and covariance data.''
The first stage has been devoted to producing the description of different adjustment methodologies and assessing
their merits. A detailed document related to this first stage has been issued. Nine leading organizations (often
with a long and recognized expertise in the field) have contributed:  ANL, CEA, INL, IPPE, JAEA, JSI, NRG, IRSN and ORNL.
In the second stage a practical benchmark exercise was defined in order to test the reliability of the nuclear data
adjustment methodology. Comparison of the results obtained by the participants and major lessons learned in the exercise
are discussed in the present paper that summarizes individual contributions which often include several original
developments not reported separately.

The paper provides the analysis of the most important results of the adjustment of the main nuclear data of 11 major
isotopes in a 33-group energy structure. This benchmark exercise was based on a set of 20 well defined integral parameters
from 7 fast assembly experiments.
The exercise showed that using a common shared set of integral experiments but different starting evaluated libraries
and/or different covariance matrices, there is a good convergence of trends for adjustments. Moreover, a significant
reduction of the original uncertainties is often observed. Using the a-posteriori covariance data, there is a strong
reduction of the uncertainties of integral parameters for reference reactor designs, mainly due to the new correlations
in the a-posteriori covariance matrix.
Furthermore, criteria have been proposed and applied to verify the consistency of differential and integral data used in the adjustment.
Finally, recommendations are given for an appropriate use of sensitivity analysis methods and indications for future work are provided.

\end{abstract}
\maketitle

\mbox{\vspace{2in}}
\clearpage
\tableofcontents

%%% DO NOT EDIT the following section enclosed by *****
%%% ***************************************************
\lhead{Methods and Issues $\dots$}
\chead{NUCLEAR DATA SHEETS}
\rhead{M. Salvatores, \etal}
\lfoot{}
\rfoot{}
\renewcommand{\footrulewidth}{0.4pt}
%%% ***************************************************

\section{Framework and Background}

\subsection{Scope and Objectives}

Nuclear data uncertainty impact studies~\cite{Aliberti_2004,Aliberti_2006,SG26_2008} have pointed out that present
uncertainty on the nuclear data should be significantly reduced, in order to get full benefit from the advanced
modeling and simulation initiatives that have been launched worldwide in recent years. Only a parallel effort in
advanced simulation and in nuclear data improvement will be able to provide designers with more general and well
validated calculation tools that would be able to meet design target accuracies. 

Tight design target accuracies, required in order to comply with safety and optimization requirements and objectives,
can only be met if very accurate nuclear data are used for a large number of isotopes, reaction types and energy ranges.

The required accuracies on the nuclear data are difficult to meet using only differential experiments, even if innovative
experimental techniques are used.

The use of integral experiments has been essential in the past to ensure enhanced predictions for power fast reactor
cores. In some cases, these integral experiments have been documented in an effective manner and associated
uncertainties are well understood. 

A combined use of scientifically based covariance data and of integral experiments can be made using advanced statistical
adjustment techniques (see, \eg~\cite{Cecchini_1964,Mitani_1972,Chaudat_1973}). These techniques can provide in a first
step adjusted nuclear data for a wide range of applications, together with new, improved covariance data and bias factors
(with reduced uncertainties) for the required design parameters, in order to meet design target accuracies.

Moreover, the role for cross section adjustment is more and more perceived as that of providing useful feedback to
evaluators and differential measurement experimentalists in order to improve the knowledge of neutron cross sections to
be used in a wider range of applications.

Despite its recognized potential, the adjustment process has raised legitimate questions related both to physical meaning
of the individual (\ie, by isotope, reaction type and energy range) ``adjustments'', and to possible compensation effects
since integral experiments depend on a large number of parameters. Moreover, the first adjustments relied on uncertainty
data essentially based on physicists' judgment and not on any formal approach. 

It has also been pointed out that there exists no clear definition of the application domain of the adjusted multigroup
data sets. When a new reactor concept is investigated, it is difficult to define what is the mathematical/physical
extrapolation method (if any) to be used together with the previously adjusted data library. In any case, since the
adjustments are performed at the multigroup level, they will also be related to the weighting function used to produce
the original multigroup library and no unique procedures can be used to transfer the adjustments from the broad group
level (where the adjustments are usually performed) to a fine group level or, even preferable, to the continuous energy
level. Similarly, self-shielding effects are not necessarily accounted for explicitly. 

Finally, the use of the a posteriori covariance matrix (both variances and correlations) is not a self-evident exercise and
in fact in many cases use is made only of a posteriori variances.

The Working Party on International Nuclear Data Evaluation Cooperation (WPEC) of the OECD Nuclear Energy Agency Nuclear Science
Committee recognized the importance of these issues and established a Subgroup (called ``Subgroup 33'') on ``Methods and issues
for the combined use of integral experiments and covariance data''. In its mandate ``it is proposed for this WPEC Subgroup to
study methods and issues of the combined use of integral experiments and covariance data, with the objective of recommending
a set of best and consistent practices in order to improve evaluated nuclear data files. Indication should be provided on
how to best exploit existing integral experiments, define new ones if needed, provide trends and feedback to nuclear
data evaluators and measurers''. 

The Subgroup activity has been completed and a final report is being assembled. The present paper summarizes and discusses
the main results and recommendations.

\subsection{Activities of the Subgroup}

In summary, the general understanding of the adjustment methods, their theory and application, suggests a number of potential
difficulties that have to be examined carefully, in order to agree on the best approach which would allow taking full benefit
from the potential of the method. This has been the general objective of the activity of the Subgroup. 

The first step of the Subgroup activity has been the compilation of a detailed report~\cite{SG33_2011} with the assessment and
comparison of the methodologies that the different participants to this Subgroup employ for adjustment of neutron cross section
data using the observed discrepancies between calculated and measured values of integral experiments. To this purpose a
documentation of the used adjustment methodologies, developed during more than three decades from the early sixties to the
late nineties, has been provided by ANL, CEA, INL, IPPE, IRSN, JAEA, JSI, NRG, and ORNL. The report also includes the
identification of merits and drawbacks of the existing adjustment methodologies, a comparison of mathematical formulation
and specific features, and the criteria used for assessing the different methodologies. 

In order to better understand the performance of these methodologies, the robustness of the results, their extrapolability and
the impact of the uncertainties (not only on nuclear data but also on experiments and on methods) it has been decided to have
the different organizations to participate to a common benchmark adjustment exercise that allows studying these specific issues.
In particular it was agreed that the main objective of the benchmark was to test different methods of nuclear data
adjustment/assimilation and different sets of covariance data, for the purpose of reducing \eg\ the design uncertainties
of a particular type of sodium-cooled fast reactor. The benchmark makes use of a single, limited set of selected integral
experiments with fast neutron energy spectra and each organization used their own calculation methods and data.

\subsection{The Adjustment Methodologies}

As indicated above, the data statistical adjustment methods principles and mathematical formulations have been compared
in a document~\cite{SG33_2011}. Most of the methods, as reminded below, use practically the same mathematical formulation
and that formulation is briefly summarized hereafter.

Let $\vec{E}=\vec{E_i}$ $(i=1,\dots,N_E)$ denote some experimental integral variables, and let
$\vec{\sigma}=\vec{\sigma}_j$ $(j=1,\dots,N_{\sigma})$ denote the multi-group parameters defining
the model used to simulate these integral experiments, and $\vec{C}(\sigma)$ the associated calculated values
to be compared with $\vec{E}$. Let $\vec{\sigma}_m$ and $M_{\sigma}$ define the a priori expectation
and covariance matrix of the multi-group parameters, and $M_E$ define the experimental covariance matrix, including
modeling covariance information when appropriate (\ie, $M_E=V_e+V_m$). The evaluation of posterior expectation and covariances
is done by finding the minimum of the following cost function (a generalised least-square)
\begin{eqnarray}
\label{Eq1}
%\lefteqn{\chi^2_{GLS} = (\vec{\sigma}-\vec{\sigma}_m)^t M^{-1}_{\sigma} (\vec{\sigma}-\vec{\sigma}_m)} \nonumber \\
\chi^2_{GLS} &=& (\vec{\sigma}-\vec{\sigma}_m)^t M^{-1}_{\sigma} (\vec{\sigma}-\vec{\sigma}_m) \nonumber \\
&+& (\vec{E}-\vec{C}(\sigma))^t M^{-1}_E (\vec{E}-\vec{C}(\sigma)).
\end{eqnarray}

Information related to integral simulations is included in the $\vec{C}$ values as well as in their derivatives
with respect to the multi-group parameters. Using a first order approximation, one can write
\begin{equation} \label{Eq2} 
\vec{C}(\sigma)=\vec{C}({\sigma}_m) + S \cdot \left(\vec{\sigma}-\vec{\sigma}_m\right),
\end{equation} 
where $S$ is a matrix ($N_E \times N_{\sigma}$) of calculated derivatives supposed to be constant (when the cross sections slightly
change). Most of the time, $S$ is referred to relative sensitivity coefficients
\begin{equation} \label{Eq3} 
S_{ij}=\frac{\delta C_i/C_i}{\delta\sigma_j/\sigma_j}.               
\end{equation} 

From a mathematical point of view, the method is quite general and has been extensively used for many kinds of inverse
problems. In the field of reactor physics, this approach has been already applied to validate and/or further improve the
nuclear data used in the simulation of thermal and fast reactors.

As indicated above, the formulations used by the different participants have been compared in~\cite{SG33_2011}.

The following observations can be made:

\begin{itemize}
\item  Seven organisations (ANL, CEA, INL, IPPE, JAEA, JSI and IRSN) apply equivalent equations for the adjustment, though
  the names of the theory differ.
\item  ORNL uses similar equations as the above organisations. However, a correction factor (F$_{m/k}$) is applied to
  the covariance matrix of integral experiments to account for the $C/E$ discrepancy.
\item  The NRG approach is completely different. It is based on the Total Monte Carlo method to produce thousands of
  TALYS-based evaluated files using MC sampling of nuclear parameters. The method used by NRG solves the inverse problem
  of nuclear data adjustment by selecting the optimal combination of random files that best reproduce all integral experiments.
\end{itemize}

\subsection{The Benchmark Exercise}

Every participant to the benchmark exercise used the same integral experiment values $(E)$ and uncertainties, but their
own calculated values $(C)$, sensitivity coefficients, and adjustment/assimilation methods. The benchmark consisted of
a three-phase exercise:

\begin{itemize}
\item  Phase I. All participants used their own initial cross sections, own nuclear data covariance, with integral
 experiment and method correlation.
\item  Phase II. Some participants used their own initial cross sections, but a different
 nuclear data covariance matrix. This step allows a better understanding of the impact of the
 nuclear data covariance on the adjustment.
\item  Phase III. Verification of the impact of the adjustments on a few ``Target Systems.'' 
\end{itemize}

Finally, the addition of a set of integral experiments allowed a test of the robustness of the previous adjustments (stress tests).

\subsubsection{Benchmark input}

In order to limit the calculation effort and to allow pointing out major trends in more clear way, the number of
isotopes to be adjusted has been limited to ten: $^{16}$O, $^{23}$Na, $^{56}$Fe, $^{52}$Cr, $^{58}$Ni, $^{235}$U, $^{238}$U,
$^{239}$Pu, $^{240}$Pu, $^{241}$Pu plus $^{10}$B for testing. On the contrary, all major reactions have been considered.
Finally, several covariance data sets have been used.

A unique energy group structure (given in Table~\ref{table1}) has also been adopted.

\begin{table}[!htb]
\caption{33 energy group structure (eV).}
\label{table1}
\begin{tabular}{c|c|c|c|c|c} \hline\hline
\multirow{2}{*}{\bf Group} & Upper & \multirow{2}{*}{\bf Group} & Upper & \multirow{2}{*}{\bf Group} & Upper\\
& Energy & & Energy & & Energy \\ \hline
\textbf{1}     & \rule{0mm}{3mm}$1.96\times 10^{7}$ & \textbf{12}    &    $6.74\times10^{4}$    & \textbf{23}    &    $3.04\times10^{2}   $ \\
\textbf{2}     & $1.00\times 10^{7}$ & \textbf{13}    &    $4.09\times10^{4}$    & \textbf{24}    &    $1.49\times10^{2}   $ \\
\textbf{3}     & $6.07\times 10^{6}$ & \textbf{14}    &    $2.48\times10^{4}$    & \textbf{25}    &    $9.17\times10^{1}   $ \\
\textbf{4}     & $3.68\times 10^{6}$ & \textbf{15}    &    $1.50\times10^{4}$    & \textbf{26}    &    $6.79\times10^{1}   $ \\
\textbf{5}     & $2.23\times 10^{6}$ & \textbf{16}    &    $9.12\times10^{3}$    & \textbf{27}    &    $4.02\times10^{1}   $ \\
\textbf{6}     & $1.35\times 10^{6}$ & \textbf{17}    &    $5.53\times10^{3}$    & \textbf{28}    &    $2.26\times10^{1}   $ \\
\textbf{7}     & $8.21\times 10^{5}$ & \textbf{18}    &    $3.35\times10^{3}$    & \textbf{29}    &    $1.37\times10^{1}   $ \\
\textbf{8}     & $4.98\times 10^{5}$ & \textbf{19}    &    $2.03\times10^{3}$    & \textbf{30}    &    $8.32\times10^{0}   $ \\
\textbf{9}     & $3.02\times 10^{5}$ & \textbf{20}    &    $1.23\times10^{3}$    & \textbf{31}    &    $4.00\times10^{0}   $ \\
\textbf{10}    & $1.83\times 10^{5}$ & \textbf{21}    &    $7.49\times10^{2}$    & \textbf{32}    &    $5.40\times10^{-1}  $ \\
\textbf{11}    & $1.11\times 10^{5}$ & \textbf{22}    &    $4.54\times10^{2}$    & \textbf{33}    &    $1.00\times10^{-1}  $ \\ \hline\hline
\end{tabular}
\end{table}

\subsubsection{Nuclear data}

The following nuclear data were explicitly considered:

\begin{itemize}
\item  Elastic scattering infinite-dilution cross section, 
\item  Total inelastic scattering infinite-dilution cross section, 
\item  Capture infinite-dilution cross section (this includes $^{10}$B(n,$\alpha $) reaction), 
\item  Fission infinite-dilution cross section, 
\item  Average prompt fission neutron multiplicity (\nub), 
\item  Normalized prompt fission neutron spectrum, 
\item  Average cosine of elastically scattered neutrons (\mub),
\item  Average delayed fission neutron multiplicity ($\bar{\nu}_d$), as an optional adjustable parameter
   (on a voluntary basis). This proposal was driven by consideration of the impact of $\bar{\nu}_d$ on the integral $C/E$
     ratio value and uncertainty for Na void reactivity (measured in dollars). When not adjusting $\bar{\nu}_d$, the
  participants should have added the corresponding uncertainty to the $C/E$ value of Na void reactivity in
  order to reduce their statistical weight. 
\end{itemize}

The spectra of inelastically scattered neutrons have not been part of the benchmark exercise. 

\subsubsection{Nuclear covariance data}

All participants used their own nuclear covariance data (Phase I, see above). However, in Phase II of the exercise,
for comparison purposes and to disentangle effects from different a priori cross sections or covariance data,
one common set of covariance data would be used by some of the participants, in addition to their own specific
sets. The 33-group COMMARA-2.0 covariance data, developed by BNL-LANL collaboration (see section III), has been
made available for that purpose.

\subsection{Selection of Integral Experiments}

The selection of fast neutron spectrum integral experiments has been based on the availability of well
documented specifications and experimental uncertainties and possibly of some indication of uncertainty
correlations. The selected experiments cover a wide range of fast neutron energy spectra, and include
critical masses, spectral indices and, when available, selected Na void reactivity coefficients.
In the notation used for spectral indices, Fij (or Cij) is the fission (or capture) rate of the isotope
23j of the element 9i (\ie, U, Np, Pu for i=2,3,4, respectively), \eg, F37 is the $^{237}$Np fission rate.
Detailed specifications on these integral experiments were taken from~\cite{ICSBEP_2007,IRPhE_2007,LeSage_2001}:

\begin{itemize}
\item Jezebel $^{239}$Pu configuration: 1 critical mass, 3 spectral indices: F28/F25, F49/F25, F37/F25,
\item Jezebel $^{240}$Pu configuration: 1 critical mass, 
\item Flattop Pu configuration: 1 critical mass, 2 spectral indices: F28/F25, F37/F25, 
\item ZPR6-7 standard configuration: 1 critical mass, 3 spectral indices: F28/F25, F49/F25, C28/F25, 
\item ZPR6-7 High $^{240}$Pu content: 1 critical mass, 
\item ZPPR-9: 1 critical mass, 3 spectral indices: F28/F25, F49/F25, C28/F25, 2 Na void configurations: central
 void and leakage-dominated configurations, 
\item JOYO: 1 critical mass. 
\end{itemize}

A specific activity has been devoted to the assessment of integral experiment covariance data that will be
described in section~\ref{Integral_covariance}.

\subsection{Corrective Factors}

The strategy proposed in order to avoid a full reanalysis of all experiments by the participants to the
adjustment exercise has been to provide corrective factors obtained as a ratio between a very detailed
(reference) calculation and a simplified one. Hence the $C/E$ (Calculated/Experimental value) is obtained as
\begin{equation}
\label{Eq4}
\frac{C}{E} =\frac{C^s C^f}{E},
\end{equation}
where $E$ is the experimental value, $C^{s}$ is the result coming from the simplified model calculation,
and $C^{f}$ is the corresponding corrective factor.

\section{Sensitivity Analysis}
\label{Sensitivity}

A major ingredient of any adjustment process is the sensitivity analysis. A number of theoretical developments
and applications have been performed since the pioneering work of Usachev~\cite{Usachev_1967} that generalized
the standard Wigner approach for critical nuclear reactors. In general these methods and approximations used in
practical applications (\eg, the use of first order sensitivity analysis) are well understood. However, the present
activity offered a unique opportunity to perform a detailed international comparison on a set of well defined
configurations. A number of issues were tackled and recommendations were formulated.

JAEA, INL, ANL, CEA and PSI have provided full sets of sensitivity coefficients for all integral parameters
considered in the benchmark. Sensitivity coefficients for the effective multiplication factor \kef\ have also been
generated in a consistent manner by JSI and IRSN. ORNL has provided data but on a different energy group structure. 

\subsection{Methodologies}

Sensitivity methods are well established, however in practical applications one has to specify if deterministic or
Monte Carlo methods have been used and a number of ambiguities/approximations have to be eliminated or well understood
in order to compare effectively the sensitivity coefficient produced by different groups. The main features of the
different methods used within the Subgroup can be summarized as follows: 

(1) For \kef, deterministic values of the sensitivity coefficients have been obtained by Standard
Perturbation-Theory (SPT) techniques using transport-theory except for JAEA that made
use of diffusion theory. More precisely, the various analyses were carried out on the basis of:

\begin{itemize}
\item SAGEP code~\cite{Ishikawa_2005}, in conjunction with JENDL-4 data at JAEA,
\item ERANOS code~\cite{Rimpault_2002} in conjunction with ENDF/B-VII.0 data at INL and ANL, and
\item ERANOS in conjunction with JEFF-3.1.1 and JEFF-3.1 data at CEA and PSI, respectively.
\end{itemize}

In addition, in conjunction with ENDF/B-VII.0 data, the code SUSD3D~\cite{Kodeli_2001} was used together with
DANTSYS~\cite{Alcouffe_1995} at JSI.

At IRSN, stochastic values of the sensitivity coefficients based on a multi-group approach have been obtained by
TSUNAMI-3D which uses an adjoint based technique and is part of SCALE~\cite{SCALE_2009} in conjunction
with ENDF/B-VII.0 data. The sensitivity coefficients take into account that a change of a given cross section
may also influence other cross sections through modifications of their shielding factors. Thus, the
TSUNAMI sensitivity coefficients are computed with the total instead of the partial derivatives as
\begin{eqnarray}
\label{Eq5}
\hspace{-3mm}\lefteqn{S_{\sigma_x^i(g)} = \frac{dk}{k}\left/\frac{d\sigma_x^i(g)}{\sigma_x^i(g)}\right. = 
\frac{\partial k}{k}\left/\frac{\partial \sigma_x^i(g)}{\sigma_x^i(g)}\right. +} \nonumber \\
& \hspace{-3mm}\sum_{j}\sum_{y}\sum_{h}\left\{\left(\frac{\partial k}{k}\left/\frac{\partial \sigma_y^i(h)}{\sigma_y^i(h)}\right.\right)
\left(\frac{\partial \sigma_y^j(h)}{\sigma_y^j(h)}\left/\frac{\partial \sigma_x^i(g)}{\sigma_x^i(g)}\right.\right)\right\}.
\end{eqnarray}

In the formula for these ``complete'' sensitivity coefficients~\cite{SCALE_2009}, the space variable has been omitted and
\eg\ $\sigma_x^i(g)$ is the cross section of nuclide $i$ for reaction $x$ in energy group $g$. The first expression on the
right hand side corresponds to the standard definition (``explicit'' term), with the additional summations (``implicit'' effects)
as an indirect term.

(2) For the reaction rates at core center relative to $^{235}$U fission (F25), \ie, F49/F25, F28/F25, F37/F25 and C28/F25,
Generalized Perturbation Theory (GPT)~\cite{Usachev_1967} has been consistently used in the deterministic calculations for
obtaining their sensitivity coefficients.

(3) Equivalent Generalized Perturbation Theory (EGPT)~\cite{Gandini_1986} has been employed for determining the sensitivity
coefficients of the void reactivity effects in ZPPR-9 (Na void Step 3 and Na void Step 5) except for JAEA that used the standard
formulation of GPT~\cite{Usachev_1967}.

\subsection{Analysis of Sensitivity Profiles}

\subsubsection{\kef}

The sensitivity coefficients of the actinides related to the two JEZEBEL bare spheres and to the sodium-cooled systems, \ie,
the two ZPR6-7 cores, ZPPR-9, and JOYO, show very good consistency among the participants, \eg\ as seen in Fig.~\ref{figure1}.
\begin{figure}[t]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure1.png}
\caption{Sensitivity profiles of $^{239}$Pu fission cross section for ZPPR-9 \kef.}
\label{figure1}
\end{figure}
The sensitivity coefficients are almost independent of the code (deterministic SAGEP, ERANOS, DANTSYS/SUSD3D and TSUNAMI-1D,
stochastic TSUNAMI-3D) and basic nuclear data (JENDL-4, ENDF/B-VII.0 or JEFF-3.1) being used. 

As far as the sensitivity coefficients for the structural materials, sodium, oxygen, and those for the scattering
reactions of the actinides (see examples in Figs.~\ref{figure2}--\ref{figure4}), consistency is also shown for most cases.

\begin{figure}[t]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure2.png}
\caption{Sensitivity profiles of $^{23}$Na elastic cross section for ZPR6-7 \kef.}
\label{figure2}
\end{figure}
\begin{figure}[t]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure3.png}
\caption{Sensitivity profiles of $^{56}$Fe inelastic cross section for ZPPR-9 \kef.}
\label{figure3}
\end{figure}
\begin{figure}[t]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure4.png}
\caption{Sensitivity profiles of $^{239}$Pu inelastic cross section for JEZEBEL \kef.}
\label{figure4}
\end{figure}

In the case of the sensitivity coefficients for the FLATTOP core, some of the energy profiles are characterized
by larger discrepancies. However, most of these discrepancies are well understood. Thus, for the JAEA solution,
it is largely attributed to the use of a 2D (r,z) model with diffusion theory instead of spherical
geometry in conjunction with transport theory used by the other participants. This is causing much larger modeling effect
for FLATTOP than for the other systems under investigation.

A further investigation was performed comparing Monte Carlo and deterministic methods.
In particular, results for the sensitivities of the multiplication factor of the FLATTOP-Pu benchmark to perturbations
of the $^{238}$U elastic cross section were investigated. The results obtained by participants to the benchmark exercise
are shown in Fig.~\ref{figure5}~(top), including the JSI deterministic solution obtained by a refined P3 approximation
for both the elastic as well as inelastic scattering cross sections. In addition, Fig.~\ref{figure5}~(bottom) displays
JSI Monte Carlo results and data taken independently from the International Criticality Safety Benchmark Evaluation
Project (ICSBEP) handbook. The results show the following features:

(a) The reference JSI deterministic values (P1 approximation) are the largest, especially above 0.3~MeV coinciding with groups 1-8, and
peak at higher energy. Using the P3 approximation largely removes this difference and leads to agreement for energies
up to about 1~MeV with the bulk of the other solutions.

(b) The continuous energy Monte Carlo solution also agrees, by bearing in mind that ``small'' sensitivity coefficients
such as for energies below 60~keV and above 5~MeV, are clearly difficult to calculate.

(c) The ORNL values provided in 238 groups are found consistent with ICSBEP, which in turn appear consistent with the
INL and ANL solutions for energies up to about 1.5~MeV, whereas the PSI values are higher near the peak around 0.5~MeV
and the KAERI values are systematically lower for energies $<0.6$~MeV.

\begin{figure}[tb]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure5a.png}
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure5b.png}
\caption{Sensitivity profiles of $^{238}$U elastic cross section for FLATTOP-Pu \kef (see text).}
\label{figure5}
\end{figure}

\subsubsection{Spectral indices F49/F25, F28/F25, F37/F25 and C28/F25}

The GPT was used by JAEA, INL, ANL, CEA, and PSI to calculate the sensitivity coefficients of spectral indices for JEZEBEL,
ZPR6-7 and ZPPR-9 configurations. The results obtained by the participants are consistent, including those for the structural
materials, oxygen and sodium in the case of ZPR6-7 and ZPPR-9. Some examples are given in Figs.~\ref{figure6}--\ref{figure8}.
\begin{figure}[bt]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure6.png}
\caption{Sensitivity profiles of $^{238}$U inelastic cross section for ZPPR-9 F28/F25 spectral index.}
\label{figure6}
\end{figure}
\begin{figure}[bt]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure7.png}
\caption{Sensitivity profiles of $^{239}$Pu inelastic cross section for JEZEBEL F37/F25 spectral index.}
\label{figure7}
\end{figure}
\begin{figure}[bt]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure8.png}
\caption{Sensitivity profiles of $^{56}$Fe inelastic cross section for ZPR6-7 C28/F25 spectral index.}
\label{figure8}
\end{figure}

\subsubsection{Reactivity effects}

The Na void (Step 3) and Na void (Step 5) in the ZPPR-9 experiment have been considered.

In general, good consistency is shown among the participants (JAEA, INL, ANL, CEA, and PSI) providing for
ZPPR-9 deterministic solutions based upon the EGPT methodology (see Fig.~\ref{figure9}) except for
JAEA (see above). However, some differences can be seen for $^{240}$Pu close to the threshold and for the
sodium inelastic scattering cross section. 

\begin{figure*}[!htb]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure9a.png}
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure9b.png}
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure9c.png}
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure9d.png}
\caption{Sensitivity profiles of various cross sections for ZPPR-9 Na voiding.}
\label{figure9}
\end{figure*}

\subsection{Summary of Major Recommendations}

Calculation of sensitivity coefficients is now part of the standard calculation routes in many modern
code systems. The agreement among results obtained by different methods is remarkable. However, a number
of recommendations can be formulated on the basis of the comparisons described above:

\begin{itemize}
\item Sensitivity coefficients calculated, \eg\ for the multiplication factor and reaction rate ratios with
 different deterministic methods and codes agree well among them, but one should be careful in specifying definitions
 and model approximations, such as \eg\ the exact detector position and volume in the case of Generalized Perturbation
 Theory (GPT) for sensitivity coefficients of reaction rate ratios.
\item ``Small'' sensitivity coefficient values are to be used with care, since in these cases errors can arise from
 numerical problems, such as those associated with the local convergence of the importance function.
\item Resonance shielding effects, which appear not too important for the present exercise, should be
  considered with appropriate algorithms.
\item Anisotropy of scattering should be accounted for at high energies when calculating sensitivity coefficients
 to elastic and probably also inelastic scattering cross sections.
\item EGPT provides a powerful tool to calculate sensitivity coefficients for reactivity effects.
\item Adjoint based and direct Monte Carlo techniques provide an interesting alternative to deterministic methods
 in particular for complex geometries. However, both methodologies are computationally very intensive. In addition,
 especially when using the direct method, care must be taken in carrying out the calculations with a sufficient
 precision. The sensitivity coefficients are computed by means of differences between two independent calculations,
 and sufficiently large perturbations of the cross sections must be introduced to ensure obtaining statistically
 significant differences of the results and at the same time avoiding non-linearity.
\end{itemize}

\section{Nuclear Data Covariances}

Nuclear data covariances represent one of the important parameters in the cross section adjustment procedure. Generally,
the covariance matrix of a scattered data set, $x_i$ ($i=1,\dots,n$) with the average value $m_{0i}=\langle x_i\rangle$,
is defined as follows~\cite{Smith_1991}:

\noindent variance
\begin{equation} \label{Eq6}
\mu_{ii}=var(x_i)=\langle(x_i-m_{0i})^2\rangle \;\text{for}\ i=1,\dots,n, \\
\end{equation}

\noindent standard deviation (STD)
\begin{equation} \label{Eq7}
\sigma_i=std(x_i)=\sqrt{var(x_i)},
\end{equation}

\noindent covariance
\begin{equation} \label{Eq8}
\begin{matrix}
\mu_{ij}=cov(x_i,x_j)=\langle(x_i-m_{0i})(x_j-m_{0j})\rangle, \\
\text{for}\ i,j=1,\dots,n \ \text{with}\ i{\neq}j,
\end{matrix}
\end{equation}

\noindent and correlation factor ($-1\leq \rho_{ij} \leq 1$)
\begin{equation}
\label{Eq9}
%\begin{matrix}
\rho_{ij}=\frac{\mu_{ij}}{\sqrt{\mu_{ii}\mu_{jj}}}=\frac{cov(x_i,x_j)}{std(x_i)std(x_j)}.
%\end{matrix}
\end{equation}

The covariance matrix must be symmetric and positive-definite. In this section, first, the methodologies used to evaluate
the nuclear data covariances are briefly reviewed. Next, some of the actual covariance data are illustrated. Comparisons
are made among JENDL-4.0 (J-4.0 hereafter~\cite{Shibata_2011,Iwamoto_2011}), and COMMARA-2.0 (C-2.0 hereafter~\cite{Herman_2011a})
which is to be used together with the ENDF/B-VII.0 central values~\cite{Chadwick_2006}, and the CEA COMAC~\cite{DeSaintJean_2011}.
More detailed comparisons can be found in Ref.~\cite{SG33_2013}.

\subsection{Covariance Data Used in the Subgroup 33 Adjustment Exercise}

Different covariance data have been developed in recent years and some of them have been used by the participants of
Subgroup 33. For the evaluation of covariance data, different techniques have been used; in particular, a generalized
least-square method to combine large experimental data bases~\cite{Poenitz_1981,Muir_1999} or the Kalman-filter method,
see for example~\cite{Kawano_1997}. The basic idea of this method is to optimize the nuclear model parameters by the
inclusion of the cross section measurement information with the Bayesian parameter estimation. Specific techniques
have been used to assess covariance data in the resonance region~\cite{Herman_2011b,Oblozinsky_2010,Little_2008,Mughabghab_2006}.
Further discussion on the different methods can be found in the quoted references and in the final report of Subgroup 33~\cite{SG33_2013}. 

Increases in computing power have made it feasible to evaluate nuclear data and their associated covariances using Monte Carlo (MC) methodology
\cite{Koning_2007,Rochman_2011}. One of the advantages of
the MC-based method is that it does not need the sensitivity of nuclear model parameters, which frees it from
the assumption of linearity. The MC method does however require a fairly large amount of computing time to obtain sufficiently small
statistical uncertainties, as well as the need of the prior model parameter uncertainty, shape and correlation for random sampling.
These methods face the difficulty to take into account deficiencies in the nuclear reaction models, and the quality and
quantity of the cross section measurements.

Table~\ref{table2} summarizes the features of various covariance data treated in the Subgroup 33 exercise. In total, 5
covariance libraries were used.

\begin{table*}[!htb]
\centering
\caption{Features of covariance data used in the Subgroup 33 adjustment exercise.}
\label{table2}
\begin{tabular}{p{1.0in}|p{1.45in}|p{1.45in}|p{1.45in}|p{1.45in}} \hline\hline
Covariance\newline library & Evaluated Isotopes  & Covariance data included &
Methodology applied to evaluate covariance data & Other features \\ \hline 
COMMARA-2.0\newline (BNL,LANL) & - 12 light nuclei (coolants and moderators)\newline
                                 - 78 structural materials and fission products\newline
                                 - 20 actinides
                               & - Reaction cross sections\newline
                                 + \nub\ for 20 actinides\newline
                                 + PFNS of $^{238,239,240}$Pu\newline
                                 + \mub\ of $^{23}$Na, $^{56}$Fe
                               & - Generalized least-square method\newline
                                 - Resonance R-matrix analysis\newline
                                 - Kalman-filter method\newline
                                 - Kernel approximation
                               & - 33-energy group structure\newline
                                 - To be used together with the ENDF/B-VII.0 central values\newline
                                 - Released in October 2010~\cite{Herman_2011a}  \\ \hline 
JENDL-4.0\newline (JAEA)     & 95 nuclides, including\newline
                                $^{10,11}$B, $^{14,15}$N,
                                $^{16}$O, $^{23}$Na, $^{48}$Ti,
                                $^{52,53}$Cr, $^{55}$Mn, $^{56}$Fe,
                                $^{59}$Co, $^{58,60}$Ni, $^{90}$Zr,
                                $^{209}$Bi, and all actinides
                             & - Reaction cross sections\newline
                               + \mub\ for 16 light nuclei and structural materials\newline
                               - Reaction cross sections\newline
                               + \nub\ + PFNS\newline
                               + \mub\ for 79 actinides
                             & - Generalized least-square method\newline
                               - Resonance R-matrix analysis\newline
                               - Kalman-filter method
                             & - ENDF standard format\newline
                               - File 33 is given for resonance energy region\newline
                               - Released May, 2010~\cite{Shibata_2011,Iwamoto_2011}\\ \hline 
COMAC(CEA)\newline
+\newline JENDL/ENDF for
some isotopes      & - 24 light and intermediate nuclei\newline
                     - 15 actinides
                   & Reaction cross sections \newline
                     + \nub\ (from JENDL-4)\newline
                     + PFNS (from JENDL-4)
                   & - Generalized least-square\newline
                     - Resonance R-matrix and Optical Model analysis\newline
                     - Marginalization of systematic experimental uncertainties 
                   & 33-energy group structure  \\ \hline
TENDL (NRG)        & \rule{0mm}{3mm}$^{235,238}$U and $^{239}$Pu
                   & No covariance files but random ENDF files based on TENDL-2010~\cite{Koning_2010}
                   & Monte Carlo-based method:\newline
                     TMC + selection based on distance minimization & Pointwise cross sections \\ \hline 
SCALE 6.1\newline (ORNL)  & - 2 isotopes in structural materials\newline
                            - 8 actinides ($^{234,235,238}$U, $^{237}$Np, $^{239,240,241,242}$Pu)
                          & Reaction cross sections\newline
                            + \nub\ + PFNS  & - Generalized least-square method\newline
                            - Delta chi-square filter method
                          & - Initially a 44-energy group structure, collapsed to a 33-group structure\newline
                            - Released in June 2011~\cite{Rearden_2011} \\ \hline\hline
\end{tabular}
\end{table*}

\subsection{Comparison of Covariance Data}

Most of the covariance data (diagonal values and by and large also correlation coefficients) used in the different
participants data sets, are comparable in magnitude. However, a closer examination points out differences, sometimes 
associated with covariance data evaluation methods. To give some examples we will first show some discrepancies
among the CEA COMAC library, the COMMARA-2.0 and the JENDL-4 covariances for $^{239}$Pu and $^{241}$Pu
fission and $^{238}$U inelastic scattering.
%In a successive paragraph, differences observed between COMMARA-2.0 and JENDL-4 data
%for three specific cross sections in specific energy ranges will be discussed in detail.

Figs.~\ref{figure10}--\ref{figure12} show the diagonal values of $^{239}$Pu and $^{241}$Pu fission and $^{238}$U inelastic scattering
cross sections. The uncertainty on the $^{239}$Pu fission cross section is small in all cases. However, there is
approximately a factor of two difference between the COMAC values and the JENDL or COMMARA values. This difference
is at the origin of very significant uncertainty value on most \kef\ of Pu-fueled integral experiments (see
section~\ref{Int_Exp_Comparison}). Also, in the case of the fission of $^{241}$Pu, there are
large differences between the COMAC values and, on the other side, the JENDL or COMMARA values in the energy region below
$\sim$10~keV. These differences in uncertainty do not play a very important role in the present exercise, but can have a
much stronger impact on core design and fuel cycle parameters of large Fast Reactor cores with softer spectra. Finally, the
difference observed in the $^{238}$U inelastic scattering cross section uncertainties has a significant impact on many key integral
parameters since it will affect the slowing down characteristics of most cores, according to core composition, conversion ratio values, \etc

\begin{figure}[hbt]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure10.png}
\caption{Relative uncertainties of $^{239}$Pu(n,f) cross sections.}
\label{figure10}
\end{figure}
\begin{figure}[hbt]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure11.png}
\caption{Relative uncertainties of $^{241}$Pu(n,f) cross sections.}
\label{figure11}
\end{figure}
\begin{figure}[hbt]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure12.png}
\caption{Relative uncertainties of $^{238}$U(n,n') cross sections.}
\label{figure12}
\end{figure}

\subsection{Further Comparison of JENDL-4.0 and COMMARA-2.0 Covariance Libraries}
\label{Covariance comparison}

One can say that in general, for the 11 isotopes treated in the Subgroup 33 exercise, that is, $^{10}$B, $^{16}$O, $^{23}$Na, $^{56}$Fe,
$^{52}$Cr, $^{58}$Ni, $^{235}$U, $^{238}$U, $^{239}$Pu, $^{240}$Pu, and $^{241}$Pu, their associated covariance data are found to
be rather similar between the two libraries, probably due to the use of similar evaluation methodology such as the full or
simplified R-matrix analysis, the Bayesian estimation connected with some theoretical nuclear model codes, or the simultaneous
evaluation for fission data of major actinides, \etc\ Furthermore, experimental cross section data from the international
library EXFOR are commonly used to fit nuclear reaction model parameters.
%Further, the cross section measurement data to fit the nuclear model
%parameters are also quite common by referring the open database, that is, EXFOR~\cite{Dupont_2011}.

However, there are some isotope-reaction-energy regions where the covariance data of the two libraries are notably different.
Three examples will be examined in detail.

\subsubsection{$^{235}$U capture in 3-300~keV energy region}

As seen in Fig.~\ref{figure13}, the standard deviation (STD) of C-2.0 is exactly $\pm$20\%, while that of J-4.0 is very small
around $\pm$2-4\%. Further, the correlations of C-2.0 are almost perfectly positive, but those of J-4.0 are only partially
positive. The difference of the capture cross section center values between ENDF/B-VII.0 and J-4.0 is around -10 to +5\% in
this energy range. From Refs.~\cite{Shibata_2011,Iwamoto_2011,Herman_2011a}, C-2.0 applied the Bayesian code KALMAN
with the GNASH code for the covariance evaluation, on the other hand, JENDL-4.0 used the generalized least square code GMA. It
seems unlikely that these two methods generate such a large difference, if both methods adopt the same experimental information.
More analysis seems to be needed to assess the covariance in the fast energy region of $^{235}$U capture reaction.

\begin{figure*}[tbh]
 \includegraphics*[width=0.85\textwidth, keepaspectratio, clip=true,trim=1mm 2.5mm 1mm 2mm]{figure13.png}
 \vspace{-3mm}
 \caption{Comparisons of JENDL-4.0 and COMMARA-2.0 covariance for $^{235}$U neutron capture.}
 \label{figure13}
\end{figure*}

\subsubsection{$^{23}$Na elastic scattering data around 2~keV}

At this energy, there appears a giant resonance peak which affects significantly the sodium-voiding reactivity in sodium-cooled
fast reactor cores. As found in Fig.~\ref{figure14}, the shape of the standard deviation is extremely different between two
libraries; that is, the minimum STD value occurs at the cross section peak energy in C-2.0, and on the contrary, the maximum
appears there in J-4.0. With a simple consideration, the trend of C-2.0 seems more natural, since the larger cross sections
would be more accurate due to the small statistical uncertainty in the measurement. The correlations are also quite different. In
the C-2.0 covariance, the 2~keV peak has no correlations with other energy, while J-4.0 is partially positive everywhere above
100~eV. The covariance of C-2.0 is evaluated by the EMPIRE/KALMAN combination, where the prior resonance model parameter
uncertainties are derived from Mughabghab~\cite{Mughabghab_2006}; on the other hand, J-4.0 applies the GMA code with some
corrections to meet the measured cross sections with the evaluated ones of J-4.0 which is based on the multi-level Breit-Wigner
formula with rather old resonance parameter values recommended by BNL in 1981. The cross section difference between ENDF/B-VII.0
and J-4.0 is -17 to +4\% around 2~keV, therefore, the difference of STDs might be reasonable if we take into account the
corrections given to J-4.0 covariance.

\begin{figure*}[tbh]
 \includegraphics*[width=0.85\textwidth, keepaspectratio, clip=true,trim=1mm 2mm 1mm 2mm]{figure14.png}
 \vspace{-3mm}
 \caption{Comparisons of JENDL-4.0 and COMMARA-2.0 covariance for $^{23}$Na neutron elastic scattering.}
 \label{figure14}
\end{figure*}

\subsubsection{$^{56}$Fe elastic scattering in 0.3-25~keV energy range}

The central values of the $^{56}$Fe elastic scattering cross section in the resonance region
from 10$^{-5}$~eV through 850~keV are almost identical for ENDF/B-VII.0 and JENDL-4.0, since the resonance
parameters adopted in both libraries are based on a common evaluation around 1990. The covariance data of
C-2.0 and J-4.0 were, however, independently evaluated. In the C-2.0 covariance, the resonance region of $^{56}$Fe up
to 850~keV was evaluated with the kernel approximation and data from Mughabghab~\cite{Mughabghab_2006}; on the other
hand, the covariance data of J-4.0 were firstly estimated from the experimental data with the GMA code. Then the
estimated variances were modified by considering the difference between the average of the experimental data and
that of JENDL-4~\cite{Shibata_2011}. The differences of the STD shapes and the correlations in Fig.~\ref{figure15}
might stem from these utterly different methodologies of their covariance evaluations, though the energy-averaged
STD values seem rather similar with each other, that is, $\pm$5.6\% in C-2.0 and $\pm$4.5-11\% in J-4.0.

\begin{figure*}[tbh]
 \includegraphics*[width=0.85\textwidth, keepaspectratio, clip=true,trim=1mm 2mm 1mm 2mm]{figure15.png}
 \vspace{-3mm}
 \caption{Comparisons of JENDL-4.0 and COMMARA-2.0 covariance for $^{56}$Fe neutron elastic scattering.}
 \label{figure15}
\end{figure*}

\section{Covariances of Experimental Integral Parameters}
\label{Integral_covariance}

\subsection{Experimental Covariance Matrix Definition}

Experimental uncertainties of an integral parameter are usually given by the experimenters in the form of
components. However, correlations between integral parameters are scarcely found in the experiment
reports; therefore, we have to estimate them from the experimental information available.
The method adopts the following three steps.

(1) Classification of Uncertainty Components to either Common or Independent (step 1). In this step one identifies
all components of the experimental uncertainties for ``Data A'' and ``Data B'' for which quantitative
values were reported, and each of them is put into a category ``Common uncertainty (\ie, the correlation factor is 1.0)
between Data A and B'', or into a category ``Independent uncertainty (\ie, the correlation factor is 0.0)''. If an uncertainty
component is judged as a mixture of common and independent uncertainties, that is, the correlation factor is not considered
as either 1.0 or 0.0, then the uncertainty component must be divided into more detailed subcomponents until the uncertainty component
becomes either common or independent uncertainty.

(2) Summing up of Common and Independent Uncertainties (step 2). The common and independent uncertainties respectively are summed-up
statistically to obtain standard deviation, $\sigma_{Total}$, the diagonal term of matrix. The statistical
treatment is justified by the assumption that all uncertainty components are already divided until there are no correlations
between any uncertainty items in the measurement of an integral parameter. The total uncertainty of a data set $A$, $\sigma_{Total,A}$,
\ie, the diagonal term of uncertainty matrix $V_e$ (see Eq.~\eqref{Eq1}), is given by the summation in quadrature of common
and independent uncertainties
\begin{equation} \label{Eq10}
\sigma_{Total,A}=\sqrt{\sigma_{Common,A}^{2}+\sigma_{Independent,A}^{2}},
\end{equation}
where $\sigma_{Common}$ is the sum of all common uncertainty components and $\sigma_{Independent}$ is the sum
of all independent uncertainty components.

(3) Evaluation of Correlation Factor (step 3). The correlation factor, non-diagonal term, of data sets $A$ and $B$,
$\rho_{A,B}$, is derived as the ratio of common uncertainties to the total uncertainties as 
\begin{equation} \label{Eq12}
\rho_{A,B}=\frac{\sum_{i}\sigma_{Common,A,i} \times \sigma_{Common,B,i}}{\sigma_{Total,A} \times \sigma_{Total,B}},
\end{equation}
where suffix $i$ is the common uncertainty components between data set $A$ and set $B$.
Steps 1 to 3 must be repeated for all matrix elements to
generate a full experimental covariance matrix as the input of adjustment exercise. Note that, for example, the correlation
factors among several sodium void reactivity measurements would be changed depending on the combination of void steps,
even in the same experimental core. 

\subsection{Full Experimental Covariance Matrix in Subgroup 33 Exercise}

Applying the above-mentioned methodology, the full covariance matrix for the 20 experiments treated in
the Subgroup 33 exercise is summarized in Table~\ref{tab:errVe}. Additional comments are given below:

(1) The correlation factors of the Reaction Rate Ratios (RRRs) in Jezebel $^{239}$Pu, Flattop and ZPR6-7 are borrowed
from those of ZPPR-9, since the denominator of the RRRs, F25, is common in these experiments, and there is scarce
information for the former three experiments to evaluate the common and independent components of the RRRs.
The F37/F25 ratio is assumed to possess similar characteristics with F28/F25 which has a threshold feature versus neutron energy.

(2) From the fuel composition tables of~\cite{Ikegami_2011,Lell_2011}, the plutonium fuel plates used in ZPR6-7, ZPR6-7
High $^{240}$Pu content and ZPPR-9 experiments were found to be the same. This means at least that the criticality of these three
cores must be correlated through the composition uncertainties. In Table~\ref{tab:errVe}, the evaluated correlation factors with the
sensitivity coefficients of core compositions are added. The correlations among other parameters of these three cores are
neglected here, since the effects of common core material to other parameters are usually small compared with that of criticality.

\subsection{Modeling Covariance Matrix}

The evaluation methodology of the modeling covariance, $V_m$ (see Eq.~\eqref{Eq1}), depends on the method
adopted to obtain the calculation value of an integral experiment. Here, we consider three kinds of methods:
(1) Continuous-energy Monte Carlo method based on the as-built experimental geometry and compositions (MC method,
hereafter), (2) Deterministic method based on the combination of the standard calculation and the corrections by
the most-detailed models (Deterministic method), and (3) Combination of the deterministic calculation
based on the simplified geometry and the correction by the Monte Carlo calculation with as-built geometry (Combined method).
The resulting matrix for the selected experiments is given in Table~\ref{tab:errVm}. A full description can be found in~\cite{SG33_2013}.

\begin{table*}[tbh]
 \caption{Experimental covariance matrix of integral parameters $V_e$ applied in the Subgroup 33 exercise. Diagonal values are 1$\sigma$ uncertainties
  in \%, non-diagonal values are correlation factors (between $-1$ and $+1$).}
 \label{tab:errVe}
 \includegraphics*[width=0.9\textwidth, keepaspectratio]{table_ve.png}
\end{table*}
\begin{table*}[tbh]
 \caption{Modeling covariance matrix $V_m$ used in the Subgroup 33 exercise (all calculations based on
  continuous-energy Monte Carlo method). Diagonal values are 1$\sigma$ uncertainties in \%, non-diagonal values are
  correlation factors (between $-1$ and $+1$).}
 \label{tab:errVm}
 \includegraphics*[width=0.9\textwidth, keepaspectratio]{table_vm.png}
\end{table*}

\section{Comparison of Integral Experiment Initial C/E's, Uncertainties and Target System Uncertainties}
\label{Int_Exp_Comparison}

\subsection{Introduction}

In this section we will inspect the results provided by the benchmark participants regarding the initial $C/E's$, uncertainties
associated to the integral experimental analysis (both experimental and calculation), as well those related to nuclear data,
and target systems uncertainties (nuclear data related). The integral experimental analysis results have been provided by
the following organizations:

\begin{itemize}
\item ANL: ENDF/B-VII.0 cross sections, MCNP5 for experiment analysis, and ERANOS system and COMMARA-2.0 covariance
 data for uncertainty analysis.
\item CEA: JEFF-3.1.1 cross sections, ERANOS/PARIS (in conjunction with corrective factors) for experimental analysis,
 and ERANOS/PARIS and COMAC covariance data for uncertainty analysis.
\item INL: ENDF/B-VII.0 cross sections, MCNP5 for experiment analysis, and SANDRA code and COMMARA-2.0 covariance
 data for uncertainty analysis.
\item IRSN: ENDF/B-VII.0 cross sections, SCALE-6 for experiment analysis, and BERING code and COMMARA-2.0 covariance
data for uncertainty analysis (only \kef\ quantities provided).
\item JAEA: JENDL-4 cross sections, MVP for experimental analysis, and SAGEP code and JENDL-4 covariance data for uncertainty analysis.
\item JSI: ENDF/B-VII.0 cross sections, DANTSYS for experiment analysis, and SUSD3D system and COMMARA-2.0 covariance data for
uncertainty analysis (only \kef\ quantities were provided with the exclusion of JOYO MK-I). P1 and S4 with the built-in
Gaussian quadrature constants were used in DANTSYS transport calculations.
\item KAERI: ENDF/B-VII.0 cross sections, DANTSYS for experiment analysis, DANTSYS/SUSD3D system and COMMARA-2.0 covariance data
for uncertainty analysis (only \kef\ quantities provided for uncertainty evaluation).
\item ORNL: ENDF/B-VII.0 cross sections, TSUNAMI-1D for experiment analysis, and TSURFER code and ORNL covariance data for
uncertainty analysis (only 1D results analyzed).
\item PSI: JEFF-3.1 cross sections, ERANOS (in conjunction with corrective factors) for experimental analysis, and ERANOS and
COMMARA-2.0 covariance data for uncertainty analysis.
\end{itemize}

In the following sections inspection and analysis of the participants' results is provided.

\subsection{C/E's}

Table~\ref{tab:E-C} compares the $(E-C)/C$ results provided by the participants expressed in \%. For all \kef\ values, with
exception of KAERI, the discrepancies between experimental and calculation results lie in a quite narrow range of
$\sim$250~pcm. This indicates that the current cross section libraries are in very good agreement for the set of
experiments selected for the exercise, that are often used for the validation of these cross section data sets. 

However, one has not to forget that \kef\ is an integral quantity and therefore the agreement can be the result of cancellation
of uncertainties. One exception is F28/F25 of ZPR6-7.

The spectral indices discrepancies are contained in a more widespread range of 4\%. It is not clear why there is little
agreement between CEA and PSI results, which both used the JEFF-3.1 cross section set. It could be due to differences
during the cell calculations with the ECCO code. It is also interesting to note that the general tendency for all cross
section sets is to underestimate the fission spectral indices results.

Finally, for sodium void results the discrepancy spread is within a $\sim$6\% range. It is interesting to observe that both
ENDF/B-VII.0 and JEFF-3.1 cross section sets have different sign between the spectral component experiment (Step 3) and the
leakage component one (Step 5), while JENDL-4 overestimates in both cases.

\subsection{Experimental and Calculation Uncertainties on C/E's}

Table~\ref{tab:CEunc} shows the (quadratic) combination of experimental and calculation uncertainty on the $C/E$ as provided by
the participants. All the participants, as indicated in the exercise, have adopted the same experimental uncertainties provided
in the original documentation of the experiment benchmark models; therefore, the observed differences have to be attributed
to the diverse assumptions the participants have taken on their calculation values. However, the following organizations have
chosen not to apply any calculation uncertainties: CEA, JSI, and KAERI. IRSN has applied a constant 0.1\% calculation uncertainty
to seven \kef, the only quantities they have considered for the exercise.

In general the uncertainties are relatively low and quite consistent among the different participants. There are a few exceptions.
For the sodium void reactivity, participants that have adopted the Monte Carlo (instead of deterministic) codes have quite high
uncertainties ($\sim$7\%). This is due to the intrinsic (statistical) difficulty that stochastic methods encounter when confronted
with small reactivity variations. It has to be kept in mind that for a reliable adjustment it is needed that both experimental
and calculation uncertainties stay as low as possible.

Low experimental uncertainties provide good quality results, thus giving credibility to the adjustment process. Low calculation
uncertainties prevent the adjustment from compensating for shortcomings present in the calculation route. In other words, if
the calculation uncertainties are high, there is the danger that the changes in the cross sections coming from the adjustment
are not physical, but the result of an artificial compensation. However, the integral data which have large calculation
uncertainty do not harm the adjusted results, since the weight of the data become small and give less effects to the cross section
changes. The risk is to underestimate the calculation uncertainty because, in such case, the cross sections are not physically
changed to force the adjusted $C/E$ values to be 1.0.

\begin{table*}[bth]
\centering
\caption{(E-C)/C results provided by the participants expressed in \%.}
\label{tab:E-C}
\centering
\begin{minipage}[h]{0.85\textwidth}
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c} \hline \hline 
Core       & Parameter      &   ANL    &   CEA    &   INL    &   IRSN   &   JAEA   &   JSI    &   KAERI  &   ORNL   &   PSI    \\ \hline 
JEZEBEL239 & \kef      &   0.014  &   0.187  &   0.014  &   0.051  &   0.130  &  -0.091  &  -0.296  &  -0.001  &   0.059  \\
JEZEBEL239 & F28/F25        &   2.351  &   0.026  &   2.354  &    -     &   3.242  &    -     &   1.630  &   2.232  &   1.791  \\
JEZEBEL239 & F49/F25        &   2.534  &   1.412  &   2.533  &    -     &   1.667  &    -     &   2.019  &   2.570  &   1.539  \\
JEZEBEL239 & F37/F25        &   1.319  &   1.526  &   1.317  &    -     &   2.103  &    -     &   2.680  &   1.244  &  -0.013  \\
JEZEBEL240 & \kef      &   0.019  &  -0.255  &   0.019  &   0.051  &   0.160  &  -0.235  &  -0.267  &  -0.079  &  -0.373  \\
FLATTOP-PU & \kef      &  -0.100  &   0.020  &  -0.097  &  -0.112  &   0.140  &  -0.953\footnote{Results are very sensitive to
$S_{N}P_{N}$ order ($S_{6}P_{1}:-0.81$, $S_{8}P_{1}:-0.48$, $S_{12}P_{1}:-0.24$, $S_{16}P_{1}:-0.15$, $S_{48}P_{3}:-0.17$).}
 &  -0.944  &  -0.255  &   0.071  \\
FLATTOP-PU & F28/F25        &   1.811  &   1.034  &   1.812  &    -     &   2.323  &    -     &   1.278  &   3.030  &   1.386  \\
FLATTOP-PU & F37/F25        &   0.446  &   1.535  &   0.442  &    -     &   0.756  &    -     &   1.469  &   0.989  &  -0.697  \\
ZPR6-7     & \kef      &  -0.040  &  -0.270  &  -0.043  &  -0.216  &  -0.527  &  -0.188  &  -0.091  &    -     &  -0.144  \\
ZPR6-7     & F28/F25        &  -0.446  &   4.207  &  -0.448  &    -     &  -3.251  &    -     &   1.183  &    -     &  -0.441  \\
ZPR6-7     & F49/F25        &   3.761  &   3.752  &   3.756  &    -     &   2.145  &    -     &  -1.320  &    -     &   4.244  \\
ZPR6-7     & C28/F25        &  -0.973  &  -0.445  &  -0.970  &    -     &  -1.643  &    -     &  -2.387  &    -     &  -0.467  \\
ZPR6-7 240 & \kef      &   0.060  &  -0.221  &   0.063  &  -0.099  &  -0.329  &  -0.171  &  -0.003  &    -     &  -0.091  \\
ZPPR-9     & \kef      &   0.080  &   0.005  &   0.078  &  -0.133  &  -0.210  &  -0.160  &  -0.133  &    -     &   0.004  \\
ZPPR-9     & F28/F25        &   2.985  &   3.664  &   2.987  &    -     &   1.750  &    -     &   0.948  &    -     &   4.104  \\
ZPPR-9     & F49/F25        &   1.956  &   2.417  &   1.958  &    -     &   0.080  &    -     &   1.131  &    -     &   2.480  \\
ZPPR-9     & C28/F25        &  -0.918  &  -0.334  &  -0.921  &    -     &  -1.845  &    -     &  -2.586  &    -     &  -0.372  \\
ZPPR-9     & Na Void Step 3 &  -1.887  &  -3.401  &  -1.884  &    -     &  -6.385  &    -     &  -4.585  &    -     &  -2.804  \\
ZPPR-9     & Na Void Step 5 &   2.752  &   3.126  &   2.754  &    -     &  -4.970  &    -     &  -6.978  &    -     &   2.759  \\
JOYO MK-I  & \kef      &   0.249  &   0.083  &   0.255  &   0.219  &   0.180  &    -     &  -0.504  &    -     &   0.056  \\ \hline \hline 
\end{tabular}
\end{minipage}
\end{table*}

\begin{table*}[bth]
\centering
\caption{Combination of experimental and calculation uncertainties on the $C/E's$ (\%).}
\label{tab:CEunc} 
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c} \hline \hline 
Core       & Parameter      &   ANL    &   CEA    &   INL    &   IRSN   &   JAEA   &   JSI    &   KAERI  &   ORNL   &   PSI    \\ \hline 
JEZEBEL239 & \kef &   0.201  &   0.200  &   0.201  &   0.224  &   0.202  &   0.200  &   0.200  &   0.200  &   0.224  \\
JEZEBEL239 & F28/F25        &   1.421  &   1.100  &   1.421  &   -      &   1.447  &   -      &   1.100  &   1.100  &   1.487  \\
JEZEBEL239 & F49/F25        &   0.949  &   1.400  &   0.949  &   -      &   1.172  &   -      &   0.900  &   0.900  &   1.345  \\
JEZEBEL239 & F37/F25        &   1.432  &   0.900  &   1.432  &   -      &   1.612  &   -      &   1.400  &   1.400  &   1.720  \\
JEZEBEL240 & \kef &   0.201  &   0.200  &   0.201  &   0.224  &   0.202  &   0.200  &   0.200  &   0.200  &   0.224  \\
FLATTOP-PU & \kef      &   0.303  &   0.300  &   0.302  &   0.316  &   0.302  &   0.300  &   0.300  &   0.300  &   0.316  \\
FLATTOP-PU & F28/F25        &   1.860  &   1.100  &   1.860  &   -      &   1.384  &   -      &   1.100  &   1.100  &   1.487  \\
FLATTOP-PU & F37/F25        &   1.432  &   1.402  &   1.432  &   -      &   1.561  &   -      &   1.400  &   1.400  &   1.720  \\
ZPR6-7     & \kef      &   0.230  &   0.230  &   0.230  &   0.251  &   0.231  &   0.230  &   0.230  &   -      &   0.250  \\
ZPR6-7     & F28/F25        &   3.499  &   2.100  &   3.499  &   -      &   3.744  &   -      &   3.000  &   -      &   3.162  \\
ZPR6-7     & F49/F25        &   2.524  &   3.000  &   2.524  &   -      &   2.541  &   -      &   2.100  &   -      &   2.326  \\
ZPR6-7     & C28/F25        &   2.683  &   2.400  &   2.683  &   -      &   2.692  &   -      &   2.400  &   -      &   2.600  \\
ZPR6-7 240 & \kef      &   0.221  &   0.220  &   0.221  &   0.242  &   0.222  &   0.220  &   0.220  &   -      &   0.241  \\
ZPPR-9     & \kef      &   0.120  &   0.117  &   0.117  &   0.154  &   0.119  &   0.117  &   0.117  &   -      &   0.153  \\
ZPPR-9     & F28/F25        &   2.915  &   2.700  &   2.915  &   -      &   3.414  &   -      &   2.700  &   -      &   2.879  \\
ZPPR-9     & F49/F25        &   2.119  &   2.000  &   2.119  &   -      &   2.338  &   -      &   2.000  &   -      &   2.236  \\
ZPPR-9     & C28/F25        &   1.992  &   1.900  &   1.992  &   -      &   2.354  &   -      &   1.900  &   -      &   2.147  \\
ZPPR-9     & Na Void Step 3 &   7.737  &   1.900  &   7.737  &   -      &   5.593  &   -      &   1.900  &   -      &   4.225  \\
ZPPR-9     & Na Void Step 5 &   7.543  &   1.900  &   7.543  &   -      &   5.311  &   -      &   1.900  &   -      &   4.133  \\
JOYO MK-I  & \kef      &   0.181  &   0.180  &   0.181  &   0.206  &   0.182  &   -      &   0.180  &   -      &   0.206  \\ \hline \hline 
\end{tabular}
\end{table*}


\subsection{Nuclear Data Uncertainties}

Table~\ref{tab:NDunc} illustrates the nuclear data uncertainties evaluated by the participants on the experimental quantities
considered for the adjustment exercise. ORNL used ENDF/B-VII.0 covariance information that was collapsed to fit the 33-group
format of the exercise. Six organizations (ANL, INL, IRSN, JSI, KAERI, and PSI) have used the same covariance data matrix,
COMMARA-2.0. As one would expect there is good agreement among the results of these organizations with the exception of KAERI.
The problem is likely related to the sensitivity coefficients (see related discussion in section~\ref{Sensitivity} devoted to this subject).

In general JENDL-4 shows consistently lower uncertainty values than COMMARA-2.0. Exceptions are the spectral indices of fission
$^{239}$Pu and $^{238}$U capture, indicating that for these two reactions JENDL-4 has larger variances than COMMARA-2.0. Uncertainties
calculated by CEA using the COMAC covariance matrix are higher than corresponding results using COMMARA-2.0. The only exception is
for $^{237}$Np fission spectral indices. Quite remarkably, the \kef\ uncertainty values span a range from $\sim$1500 to $\sim$2000~pcm.
Relatively high values (up to more than 10\%) are associated to the $^{238}$U fission spectral indices. 

Finally, concerning the uncertainties attached to the sodium reactivity coefficients, there is a fairly good agreement among all the
participants. ZPPR-9 Step 3 results lie in the range between 6 and 7\%, and Step 5 results between 7 and 10\%.

\begin{table*}[bth]
\centering
\caption{Uncertainties due to nuclear data (\%).}
\label{tab:NDunc} 
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c} \hline\hline 
Core       & Parameter      &   ANL    &   CEA    &   INL    &   IRSN   &   JAEA   &   JSI    &   KAERI  &   ORNL   &   PSI    \\ \hline 
JEZEBEL239 & \kef      &   0.640  &   2.072  &   0.636  &   0.611  &   0.693  &   0.658  &   0.117  &   1.186  &   0.511  \\
JEZEBEL239 & F28/F25        &   3.720  &   7.342  &   3.696  &   -      &   3.198  &   -      &   -      &   3.311  &   2.426  \\
JEZEBEL239 & F49/F25        &   0.830  &   2.811  &   0.823  &   -      &   0.625  &   -      &   -      &   0.809  &   0.719  \\
JEZEBEL239 & F37/F25        &   2.390  &   1.483  &   2.354  &   -      &   1.505  &   -      &   -      &   7.204  &   1.604  \\
JEZEBEL240 & \kef      &   0.660  &   1.763  &   0.656  &   0.540  &   0.649  &   0.634  &   0.123  &   0.982  &   0.579  \\
FLATTOP-PU & \kef      &   0.760  &   1.913  &   0.764  &   0.519  &   1.257  &   0.719  &   0.123  &   1.128  &   0.829  \\
FLATTOP-PU & F28/F25        &   3.120  &   7.885  &   3.093  &   -      &   2.936  &   -      &   -      &   2.616  &   1.948  \\
FLATTOP-PU & F37/F25        &   2.050  &   1.585  &   2.034  &   -      &   1.444  &   -      &   -      &   7.076  &   1.421  \\
ZPR6-7     & \kef      &   0.970  &   1.586  &   0.968  &   0.972  &   0.816  &   0.943  &   0.313  &   -      &   0.972  \\
ZPR6-7     & F28/F25        &   6.400  &   9.959  &   6.395  &   -      &   4.819  &   -      &   -      &   -      &   6.400  \\
ZPR6-7     & F49/F25        &   0.840  &   2.378  &   0.836  &   -      &   1.147  &   -      &   -      &   -      &   0.833  \\
ZPR6-7     & C28/F25        &   1.510  &   3.916  &   1.512  &   -      &   2.004  &   -      &   -      &   -      &   1.493  \\
ZPR6-7 240 & \kef      &   0.970  &   1.559  &   0.971  &   0.970  &   0.812  &   0.948  &   0.311  &   -      &   0.973  \\
ZPPR-9     & \kef      &   1.180  &   1.666  &   1.191  &   1.202  &   0.902  &   1.183  &   0.384  &   -      &   1.203  \\
ZPPR-9     & F28/F25        &   7.850  &  10.688  &   7.896  &   -      &   5.277  &   -      &   -      &   -      &   7.742  \\
ZPPR-9     & F49/F25        &   0.870  &   2.387  &   0.870  &   -      &   1.152  &   -      &   -      &   -      &   0.846  \\
ZPPR-9     & C28/F25        &   1.550  &   3.894  &   1.545  &   -      &   2.030  &   -      &   -      &   -      &   1.521  \\
ZPPR-9     & Na Void Step 3 &   7.670  &   6.493  &   7.563  &   -      &   5.950  &   -      &   -      &   -      &   7.228  \\
ZPPR-9     & Na Void Step 5 &   9.920  &   8.543  &   9.679  &   -      &   7.311  &   -      &   -      &   -      &   9.157  \\
JOYO MK-I  & \kef      &   0.890  &   1.416  &   0.863  &   0.867  &   0.583  &   -      &   0.348  &   -      &   0.878  \\ \hline\hline 
\end{tabular}
\end{table*}

\subsection{Uncertainty Consistency for Adjustment}

In this section we examine the consistency of the combined experimental, calculation, and nuclear data uncertainties with the
observed discrepancies between experimental and calculated results. In order to establish this consistency, we define the
adjustment margin $AM$
\begin{equation} \label{Eq13} 
AM=U_{ND}+U_{C/E}-\left|\frac{E-C}{C}\right|,
\end{equation} 
where $U_{ND}=\sqrt{SM_{\sigma}S^t}$ is the uncertainty associated to nuclear data, $U_{C/E}=\sqrt{V_e+V_m}$ is the quadratic
combination of the experimental and calculation uncertainty, $E$ is the experimental result and $C$ the calculated result.
The $AM$ quantity establishes if in the adjustment there is enough room provided by the nuclear data uncertainty to accommodate
the $C/E$ discrepancy. Of course, the $C/E$ discrepancy has to take into account its associated uncertainty. If the $AM$ values
are negative, this implies that there is not enough uncertainty for the adjustment in the one sigma range. This will be reflected,
afterwards, in the ${\chi}^{2}$ values. One could interpret the appearance of negative values as a sign of some inconsistency in
the covariance matrix (usually due to too low uncertainties associated to specific cross sections).

The $AM$ quantity is similar to the $\chi$ quantity used by JAEA as explained in~\cite{SG33_2011}. The $\chi$ for every integral
parameter is defined as
\begin{equation} \label{Eq14}
\chi=\frac{|E-C|}{\sqrt{U_{ND}^2+U_{C/E}^2}}
\end{equation} 
and is used with a three sigma range criterion for deciding the elimination of an experiment from the adjustment.

Moreover, other methods can be suggested to verify the consistency of a set of integral experiments and parameter
uncertainties. For example, the Cook's distance is used in statistics to estimate the influence of a data point when
performing least squares analysis. Data points with large residuals (outliers) and/or high uncertainties may distort
accuracy of the adjustment as well as its conclusions. Points with a large Cook's distance are to be carefully
checked. Some results of this type of analysis can be found in~\cite{SG33_2013} and in section~\ref{Cook distance}.

The present analysis has been performed using the rather simple and intuitive formulation of Eq.~\eqref{Eq13}.

Table~\ref{tab:AM} reports the $AM$ values for the different solutions provided by the participants.

\begin{table*}[bth]
\centering
\caption{Adjustment Margin $(AM)$ values expressed in \%.}
\label{tab:AM} 
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c} \hline\hline 
Core       & Parameter&    ANL     &    CEA    &    INL &   IRSN   &   JAEA   &   JSI    &   KAERI   &   ORNL   &   PSI  \\ \hline 
JEZEBEL239 & \kef &   0.827   &   2.085   &   0.823   &   0.784  &   0.764  &   0.767  &   0.021   &   1.385        &   0.676  \\
JEZEBEL239 & F28/F25  &   2.790  &   8.417   &   2.763  &    -    &   1.404  &    -    &    -    &   2.179      &   2.122   \\
JEZEBEL239 & F49/F25  &\cgs -0.755 & 2.799 & \cgs -0.761 & - & 0.129 & - & - &\cgs -0.861 & 0.525 \\
JEZEBEL239 & F37/F25  &  2.503  &  0.857  &  2.469  & - &  1.014 & - & - &  7.360 & 3.311 \\
JEZEBEL240 & \kef & 0.842 &  1.708  &  0.83 & 0.726 & 0.691 & 0.598 & 0.056 & 1.103 & 0.430 \\
FLATTOP-PU & \kef & 0.963 & 2.193 & 0.969 &  0.723 & 1.418 &  0.602 & \cgs -0.521 & 1.173  &  1.075 \\
FLATTOP-PU & F28/F25  & 3.169 & 7.950 & 3.141 & - & 1.997 &  - &  -  & 0.686 &  2.048 \\
FLATTOP-PU & F37/F25  &  3.036 &  1.452 &  3.024 &  - & 2.249 & - & - &  7.487 & 2.444\\
ZPR6-7     & \kef  &  1.160 &  1.545 & 1.155 &  1.008 &  0.520 & 0.985 &  0.453 & -  &  1.079 \\ 
ZPR6-7     & F28/F25   &  9.453 &  7.852  &  9.446  &  -   &  5.312 &  - &  -  &  - &  9.121  \\
ZPR6-7     & F49/F25   & \cgs -0.397 &  1.626  & \cgs -0.396 &  -   & 1.543 & -  &  -  &  - & \cgs -1.085 \\ 
ZPR6-7     & C28/F25 &  3.220   &  5.871 & 3.225 &  -  & 3.053 & - &  - &  - &  3.6 \\
ZPR6-7 240 & \kef &  1.131  &  1.559 & 1.129  &  1.116 &  0.705 &  0.997 & 0.528 & - &  1.123  \\
ZPPR-9     & \kef    &   1.220   &   1.778   &   1.230    &   1.224  &   0.811  &   0.957  &   0.368 &  - &  1.352 \\
ZPPR-9     & F28/F25  &  7.780 &  9.724 & 7.825 & - &  6.941 & - & - & - & 6.516 \\
ZPPR-9     & F49/F25  &  1.033 &  1.970 & 1.031 &   - &  3.410 & - &  - &  - &  0.602 \\
ZPPR-9     & C28/F25  &  2.625 &  5.460 &  2.616 & - & 2.539 & - & - & - & 3.296 \\
ZPPR-9     & Na Void Step 3 & 13.520 &  4.995 & 13.416  &  -  & 5.158 & - &  -  &  - & 8.649  \\
ZPPR-9     & Na Void Step 5 &  14.711 & 7.317 & 14.468 & - &  7.653 &  - &  - & - & 10.531 \\
JOYO MK-I  & \kef & 0.822 &  1.513 & 0.789 & 0.854 & 0.585 &  - & 0.024 &  - & 1.029 \\ \hline\hline 
\end{tabular}
\end{table*}

There are only seven negative values (indicated in shaded cells). The KAERI FLATTOP \kef\ would be challenging to adjust in
view of the associated quite large negative $AM$ (-521~pcm). This is the result of different contributions: a quite unusually
large initial $C/E$ discrepancy (almost 1000~pcm), no calculation uncertainty provided, and, finally, the sensitivity coefficient
problem previously mentioned. PSI has a relative high negative $AM$ for the $^{239}$Pu fission spectral index in ZPR6-7, and one
should expect some difficulty in adjusting this integral parameter, but this cannot be proven as PSI has not yet performed the adjustment.

The ANL and INL four negative values are relative to the $^{239}$Pu fission spectral indices and are likely due to the low
uncertainty associated by COMMARA-2.0 to $^{239}$Pu fission. The only negative value for CEA is associated to the fairly low
uncertainty on $^{237}$Np fission for COMAC previously observed in section V.4. For these entire five negative $AM$ values one
should expect, after adjustment, a difficulty for the $C/E$ to reach the unity value even including the new evaluated associated
uncertainty in the one sigma range. This is indeed the case if one inspects such quantities after adjustment (see related section).
The impact on ${\chi}^{2}$ is not expected to be large for two reasons. First the negative $AM$ values are relatively low ($<$1\%
for spectral indices), and then, because the ${\chi}^{2}$is normalized to (divided by) the number of experiments (degrees of
freedom). Having only one or two small inconsistencies will not show up in an adjustment with 20 experiments.

Let us continue our analysis observing that in Eq.~\eqref{Eq13}, a not negligible role is played by the $U_{C/E}$ term. As indicated
in section V.3, the desirable situation is to have this quantity as low as possible in order to provide a reliable adjustment. Let's
define the experiment merit $EM$, where in Eq.~\eqref{Eq13} we suppress the term $U_{ND}$. What we want to spot now are positive
values. Positive $EM$ will indicate that the experiment is not providing enough useful information (\ie, has not enough merit to
be included in the adjustment) because there is too much uncertainty associated with respect to the observed $C/E$. In practice,
these experiments could be excluded from the adjustment because they are not valuable either for poor experimental quality and/or
because the employed calculation analysis carries too much uncertainty, or kept in order to provide a constraint that should not
be changed by the adjustment. 

In total fairness, in terms of usefulness of an experiment, correlations in the experimental uncertainties should be established
and one should look also at the case where only the experimental uncertainty is considered. In fact, the calculation uncertainty
component depends on circumstances independent from the experiment. Besides, another criterion for retaining or discarding an
experiment would be to look at their correlation through the cross product of sensitivities weighted with the covariance matrix
(the so-called representativity factor). If the correlation factor is very close to 1, one of the two experiments should be
discarded as it provides redundant information (unless the configurations are not experimentally correlated), and kept only for
a posteriori verification with the new adjusted cross sections and covariance matrix data. 

On the other hand, because $EM$ values neglect the uncertainty coming from the nuclear data $U_{ND}$, this could, in some
circumstances, mislead in concluding about the usefulness of an experiment. In particular, let's consider the case where there
is a very good agreement between calculation and experimental results so that the $(E-C)/C$ is almost zero, and, therefore,
$EM$ is positive. If the initial nuclear data uncertainties are greater than the $U_{C/E}$, still the resulting a posteriori
uncertainty will be reduced, after adjustment, for the experiment under consideration. Therefore, one has to be very careful
in drawing conclusions, and, for sure, this subject of experimental selection deserves further investigations.

$EM$ values corresponding to the benchmark solutions provided by participants are shown in Table~\ref{tab:EM}. Inspection
of this table provides the following conclusions regrouped by type of integral parameters:

\begin{itemize}
\item Critical Masses -- For organizations using ENDF/B-VII.0 cross sections only the JOYO experiment provides a useful
 contribution to the adjustment. This can be readily seen looking at the column of IRSN that has considered only critical
 masses. The reason for this is the excellent performance of ENDF/B-VII.0 for Pu fuelled fast cores, while the significant
 amount of $^{235}$U in JOYO requires an adjustment (in particular for the capture, as it will be seen in the section~\ref{Adjustment}
 on adjustment). On the contrary, organizations using JEFF-3.1 (CEA, and PSI) and JENDL-4 (JAEA) perform well for JOYO. CEA will
 take advantage of adjustment associated to the discrepancies in \kef\ of JEZEBEL $^{239}$Pu (and strangely enough not
 $^{240}$Pu), FLATTOP but not for the softer cores (ZPR6-7, ZPPR-9). Exactly the contrary is true for JAEA (good agreement
 for harder cores, adjustment needed for softer cores). It is not clear why for PSI, that uses JEFF-3.1, the Pu fuelled cores
 seem not to need any adjustment.
\item Spectral Indices -- $^{238}$U capture spectral indices have positive $EM$ values for everybody (with exception of
 KAERI), and, therefore, it seems not to provide any contribution to the adjustment except as a constraint. For the fission
 spectral indices the situation is more complex. $^{238}$U fission spectral indices require adjustments in harder cores
 practically for all participants, while JEFF-3.1 needs it also for softer cores. $^{239}$Pu fission indices need adjustment
 in all cores. One exception is JAEA that has positive $EM$ values for the ZPR6-7 and ZPPR-9 cores, indicating that the adjustment
 needed for the Pu fuelled softer cores \kef\ do not concern the $^{239}$Pu fission but some other reactions (or isotope).
 For $^{237}$Np fission spectral indices the situation is mixed. These spectral indices are not important for the adjustment of
 ENDF/B-VII.0 users, while action is needed for CEA on both JEZEBEL and FLATTOP, and JAEA (but only for JEZEBEL). The fact that
 PSI has positive $EM$ values for these indices indicate that probably is using a version of JEFF-3.1 with different $^{237}$Np
 cross sections with respect to CEA.
\item Sodium Void Reactivity Coefficients -- ENDF/B-VII.0 does not benefit from these experiments, the main reason being the high
 uncertainties associated to the $C/E$'s. JAEA will use the information from the experiment dominated by the central component,
 while CEA will also benefit from that dominated by the leakage component. There is no agreement on the $EM$ values of CEA and
 PSI. This time the indication is that they are probably using different $^{23}$Na cross sections.
\end{itemize}

\begin{table*}[bth]
\centering
\caption{Experiment Merit $(EM)$ values expressed in \%.}
\label{tab:EM} 
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c} \hline\hline 
Core       & Parameter      &  ANL    &  CEA    &   INL   &  IRSN   &  JAEA   &   JSI   &  KAERI  &  ORNL   &  PSI    \\ \hline 
JEZEBEL239 & \kef      &  0.187  &  0.013  &  0.187  &  0.173  &  0.072  &  0.109  & -0.096  &  0.199  &  0.165  \\
JEZEBEL239 & F28/F25        & -0.930  &  1.074  & -0.933  &   -     & -1.795  &   -     & -0.530  & -1.132  & -0.305  \\
JEZEBEL239 & F49/F25        & -1.585  & -0.012  & -1.584  &   -     & -0.496  &   -     & -1.119  & -1.670  & -0.193  \\
JEZEBEL239 & F37/F25        &  0.113  & -0.626  &  0.115  &   -     & -0.491  &   -     & -1.280  &  0.156  &  1.707  \\
JEZEBEL240 & \kef      &  0.182  & -0.055  &  0.182  &  0.186  &  0.042  & -0.035  & -0.067  &  0.121  & -0.149  \\
FLATTOP-PU & \kef      &  0.203  &  0.280  &  0.205  &  0.204  &  0.162  &  0.188  & -0.644  &  0.045  &  0.245  \\
FLATTOP-PU & F28/F25        &  0.049  &  0.066  &  0.048  &   -     & -0.939  &   -     & -0.178  & -1.930  &  0.100  \\
FLATTOP-PU & F37/F25        &  0.986  & -0.133  &  0.990  &   -     &  0.805  &   -     & -0.069  &  0.411  &  1.023  \\
ZPR6-7     & \kef      &  0.190  & -0.040  &  0.187  &  0.035  & -0.296  &  0.042  &  0.139  &   -     &  0.106  \\
ZPR6-7     & F28/F25        &  3.053  & -2.107  &  3.051  &   -     &  0.493  &   -     &  1.817  &   -     &  2.721  \\
ZPR6-7     & F49/F25        & -1.237  & -0.752  & -1.232  &   -     &  0.396  &   -     &  0.780  &   -     & -1.918  \\
ZPR6-7     & C28/F25        &  1.710  &  1.955  &  1.713  &   -     &  1.050  &   -     &  0.013  &   -     &  2.133  \\
ZPR6-7 240 & \kef      &  0.161  & -0.001  &  0.158  &  0.143  & -0.107  &  0.049  &  0.217  &   -     &  0.150  \\
ZPPR-9     & \kef      &  0.040  &  0.112  &  0.039  &  0.021  & -0.091  & -0.226  & -0.016  &   -     &  0.149  \\
ZPPR-9     & F28/F25        & -0.070  & -0.964  & -0.071  &   -     &  1.664  &   -     &  1.752  &   -     & -1.225  \\
ZPPR-9     & F49/F25        &  0.163  & -0.417  &  0.161  &   -     &  2.257  &   -     &  0.869  &   -     & -0.244  \\
ZPPR-9     & C28/F25        &  1.075  &  1.566  &  1.071  &   -     &  0.509  &   -     & -0.686  &   -     &  1.775  \\
ZPPR-9     & Na Void Step 3 &  5.850  & -1.501  &  5.853  &   -     & -0.792  &   -     & -2.685  &   -     &  1.421  \\
ZPPR-9     & Na Void Step 5 &  4.791  & -1.226  &  4.789  &   -     &  0.341  &   -     & -5.078  &   -     &  1.374  \\
JOYO MK-I  & \kef      & -0.068  &  0.097  & -0.074  & -0.013  &  0.002  &   -     & -0.324  &   -     &  0.150  \\ \hline\hline 
\end{tabular}
\end{table*}

Conversely to $EM$ let's now define the Theoretical Adjustment Margin (\textit{TAM}) where in Eq.~\eqref{Eq13} we suppress the
$U_{C/E}$ term. This corresponds to the ideal situation where we have perfect measured experiment and perfect calculation tools
with no error or uncertainty associated. Even though this is more an academic exercise, the negative \textit{TAM} values can provide
stronger recommendations for the quality of the covariance data. Again negative \textit{TAM} will point out the inability of the
covariance data to accommodate the adjustment, where now all the discrepancies have to be attributed to shortcomings in the
nuclear data. Table~\ref{tab:TAM} shows the \textit{TAM} values for the solutions provided by the participants. As expected, there are
more negative \textit{TAM} values than $AM$ ones. If one excludes the columns of KAERI and ORNL (very low or zero $U_{ND}$ values for
the reasons previously indicated), it is quite striking that the rows for the $^{239}$Pu fission spectral indices have practically
all negative \textit{TAM} values. This indicates that all cross section files have overly optimistic uncertainties for $^{239}$Pu
fission cross sections.

\begin{table*}[bth]
\centering
\caption{Theoretical Adjustment Margin (\textit{TAM}) values expressed in \%.}
\label{tab:TAM} 
\begin{tabular}{c|c|c|c|c|c|c|c|c|c|c} \hline\hline
Core       & Parameter &  ANL &   CEA  &   INL   &  IRSN   &  JAEA   &  JSI   &  KAERI  &   ORNL  &   PSI  \\ \hline
JEZEBEL239 & \kef    &  0.626 &  1.885   &  0.622  &  0.560  &  0.562  & 0.567  &\cgs -0.179&  1.185     &  0.453                    \\
JEZEBEL239 & F28/F25 &  1.369 &  7.317   &  1.342  &   -     &\cgs -0.043&  -     &   -      &  1.079   &  0.635 \\
JEZEBEL239 & F49/F25 &\cgs -1.704  &  1.399  &\cgs -1.710&  -     &\cgs -1.042&  -   &  -  &\cgs -1.761 & \cgs -0.820\\
JEZEBEL239 & F37/F25 &  1.071 & \cgs -0.043&  1.037     &   -    & \cgs -0.599&  -     &  -   &  5.960  &  1.591  \\
JEZEBEL240 & \kef    &  0.641 &  1.508   &  0.637  &  0.502  &  0.489  & 0.398  &\cgs -0.144&  0.903    &  0.207  \\
FLATTOP-PU & \kef    &  0.660 &  1.893   &  0.667  &  0.407  &  1.116  & 0.462  &\cgs -0.821&  0.873    &  0.759  \\
FLATTOP-PU & F28/F25 &  1.309 &  6.850   &  1.281  &   -     &  0.613  &  -     &   -  &\cgs -0.414&  0.562  \\
FLATTOP-PU & F37/F25 &  1.604 &  0.050   &  1.592  &   -     &  0.688  &  -     &   -  &  6.087  &  0.724    \\
ZPR6-7     & \kef    &  0.930 &  1.315   &  0.925  &  0.757  &  0.289  & 0.755  &  0.223         &   -       &  0.828   \\
ZPR6-7     & F28/F25 &  5.954 &  5.752   &  5.947  &   -     &  1.568  &  -     &   -  &   -     &  5.958    \\
ZPR6-7     & F49/F25 &\cgs -2.921 & \cgs -1.374 & \cgs -2.920&   -  & \cgs-0.998 &  -  & - & - &\cgs -3.411\\
ZPR6-7     & C28/F25 &  0.537 &  3.471   &  0.542  &   -     &  0.361  &  -     &   -  &   -     &  1.026    \\
ZPR6-7 240 & \kef    &  0.910 &  1.339   &  0.908  &  0.871  &  0.483  & 0.777  &  0.308         &   -       &  0.882   \\
ZPPR-9     & \kef    &  1.100 &  1.661   &  1.113  &  1.069  &  0.692  & 0.840  &  0.251         &   -       &  1.198   \\
ZPPR-9     & F28/F25 &  4.865 &  7.024   &  4.909  &   -     &  3.526  &  -     &   -  &   -     &  3.637    \\
ZPPR-9     & F49/F25 & \cgs -1.086 & \cgs -0.030 & \cgs -1.088 &   -  &  1.072 & - &  - & -  & \cgs -1.634\\
ZPPR-9     & C28/F25 &  0.632 &  3.560   &  0.624  &   -     &  0.185  &  -     &   -  &   -     &  1.149    \\
ZPPR-9     & Na Void Step 3 &  5.783     &  3.095  &  5.679  &   -     &\cgs -0.434&  -      &   -  &   -  &  4.424                    \\
ZPPR-9     & Na Void Step 5 &  7.168     &  5.417  &  6.925  &   -     &  2.341 &  -   &   -     &   -       &  6.398         \\
JOYO MK-I  & \kef    &  0.641 &  1.333   &  0.608  &  0.648  &  0.402  &  -     &\cgs -0.156 &   -   &  0.823    \\ \hline\hline
\end{tabular}
\end{table*}

Another good reason for looking at \textit{TAM} values is the following consideration: if the method uncertainty is very
large, the $AM$ and $\chi$, quoted above, would suggest that an experiment is still useful for the adjustment, despite the
fact that the use of better methods (\eg, with fewer approximations) would make clear that the specific experiment is not
useful. In other words, why should one compensate weaknesses of the calculation methods with cross section modifications? This
was an early criticism to any statistical adjustment method. 

One should simply remember that the method uncertainty reduction is a necessary condition in order to provide unambiguous indications
on the integral experiment selection. This is also consistent with the fact that in most cases what we call ``method uncertainty''
is in fact more a systematic uncertainty or even a bias, with rather limited ``statistical'' meaning. This is especially true when
deterministic methods are used in the experiment analysis. When stochastic methods are used the uncertainty is more statistical in
nature, provided that the Monte Carlo code employed in the analysis has no errors or computational approximations (such as the
treatment of unresolved resonances in early versions of MCNP4).

\section{Analysis of the Adjustments}
\label{Adjustment}

In this section the results of the adjustments are discussed. First the so-called Phase I (\ie, all participants use their own
initial cross sections, own nuclear data covariance, with integral experiment and method correlation) results are investigated
with focus on specific cases of interest. A typical result of Phase II (\ie, use of own initial cross sections, but a different
nuclear data covariance matrix) is then discussed. The impact of adjustments and of a posteriori correlations (Phase III) is
successively analyzed.

Finally, the results of some ``stress tests'' performed in order to verify the robustness of the adjustment procedures, are
also reported.

\subsection{Use of Different Nuclear Data and Associated Covariances (Phase I)}

\subsubsection{General comments}

A few general comments can be made.

In most results, no incoherent trends are found when compared to initial uncertainties. Fig.~\ref{figure16} illustrates this
major effect for some typical examples.

\begin{figure*}[bth]
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure16a.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure16b.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure16c.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure16d.png}
\caption{Example of adjustment trends for major isotopes. The results are consistent with a priori and a posteriori uncertainties.}
\label{figure16}
\end{figure*}

One other important point is that variances are not that much reduced between prior and posterior cross sections
uncertainties (except for CEA where initial uncertainties for some isotopes may be found pessimistic and some minor cross
sections). It means that the final overall uncertainty reductions on benchmarks calculation or concept calculation are mainly
due to correlations created by the adjustment. This point will be investigated in the next paragraph.

A final additional general comment on these results is the fact, that it appears that only a few ($\sim$5) initial isotopes
are involved in the adjustment procedure. $^{58}$Ni, $^{52}$Cr, $^{10}$B do not contribute to the data adjustment.

\subsubsection{Specific adjustment analysis}

A first investigation of the adjustment results for some selected data as obtained by JAEA (using their own nuclear data and
covariance data set J-4.0), INL (using their own nuclear data and covariance data set COMMARA-2.0) and CEA (using their own nuclear
data and covariance data set COMAC) is given~below:

(a) $^{239}$Pu capture (Figs.~\ref{figure17}--\ref{figure19})

\begin{itemize}
\item In the energy range $\sim$3-500~keV the uncertainties in the three covariance data sets (J-4.0, COMAC,
COMMARA-2.0) are rather similar and the uncertainty is in the range $\sim$6-9\% to 7-12\%. As for the adjustments, they are
consistently indicating an increase of the capture cross section by $\sim$1-2\% up to a maximum of $\sim$10\%.
\item In the range $\sim$5-50~keV, the suggested increase of the capture cross sections in the three files are such that
even the adjusted data look very consistent and still close to each other as before adjustment.
\end{itemize}

\begin{figure}[bth]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure17.png}
\caption{INL $^{239}$Pu neutron capture adjustments.}
\label{figure17}
\end{figure}
\begin{figure}[bth]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure18.png}
\caption{CEA $^{239}$Pu neutron capture adjustments.}
\label{figure18}
\end{figure}
\begin{figure}[bth]
\includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure19.png}
\caption{JAEA $^{239}$Pu neutron capture adjustments.}
\label{figure19}
\end{figure}

(b) $^{239}$Pu inelastic scattering cross section and prompt fission neutron spectrum (Fig.~\ref{figure20}).

\begin{itemize}
\item $^{239}$Pu(n,n') is an important reaction (\ie, large sensitivities) when $^{239}$Pu is a major component
 of the core (\eg, JEZEBEL and FLATTOP), in particular for \kef\ and F28/F25.
\item In general, some decrease of (n,n') is suggested for each file. For example,
 this trend allows to get a better agreement with F28/F25, which is underestimated for both JEZEBEL and FLATTOP by
 all groups, since negative sensitivity coefficients for the F28/F25 parameter to variations of the $^{239}$Pu(n,n')
 are calculated as expected (even if not in perfect agreement) by all groups.
\item However, the changes of the inelastic cross section are the highest for INL and the lowest for CEA.
\end{itemize}

Even if after adjustment a rather better agreement is found among the three data sets, a good understanding of the
trends is obtained only if the $^{239}$Pu inelastic cross section adjustments are considered together with the
prompt fission neutron spectrum adjustments. In fact, only JAEA and CEA results show significant adjustments for that
parameter while INL and ANL show very small adjustments. The lower adjustment (decrease) of (n,n') in the JAEA
and CEA results is partly compensated by the decrease of the energy fission spectrum below $\sim$3.5~MeV suggested both
in the JAEA and in the CEA results. When a smaller decrease of the inelastic cross section is suggested, this is
associated with a higher reduction of the prompt fission neutron spectrum (see CEA with respect to JAEA adjustments
of the two parameters). In fact a lower inelastic cross section at high energy allows fewer neutrons below, \eg\ 1-2~MeV,
and the same effect is produced by a lower amount of prompt fission neutrons in that energy range. In other words, since
both adjustments of the JAEA and CEA results (\ie, decrease of (n,n') and harder fission spectrum) consistently harden
the neutron spectrum and help to improve the $C/E$ of F28/F25 in FLATTOP and JEZEBEL, in the INL adjustment a larger (n,n')
decrease is needed since a smaller decrease (JAEA and CEA) is ``compensated'' by the fission spectrum hardening.
In summary, the net result is better $C/E$ values (\ie, closer to 1) for the FLATTOP and JEZEBEL F28/F25 (and F37/F25) with
the three adjustments (JAEA, CEA and INL). Finally, the suggested change of the prompt neutron fission spectrum both in
the JAEA and in the CEA cases, are very consistent.

\begin{figure*}[tbh]
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure20a.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure20b.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure20c.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure20d.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure20e.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure20f.png}
\caption{$^{239}$Pu(n,n') cross section and prompt fission spectrum adjustments.}
\label{figure20}
\end{figure*}

(c) $^{238}$U inelastic scattering and fission cross sections (Figs.~\ref{figure21}--\ref{figure23}).

\begin{itemize}
\item The uncertainty values for the inelastic scattering cross section in the three files are rather different in
 magnitude and energy trend. For example, in the range 1-20~MeV the COMMARA-2.0 uncertainties are 2-3 times higher
 than in the J-4.0 (and in the CEA-COMAC) covariance dataset. Below $\sim$1~MeV that trend is inversed between the two files.
\item The $^{238}$U inelastic cross section adjustments of the present benchmark are in any case rather small and
 often much smaller than the uncertainties, and the a posteriori cross sections are only marginally more consistent
 than the a priori values. Anyway CEA and INL/ANL exhibit the need to reduce this reaction.
\item For CEA, there is an adjustment of the fission cross section of $^{238}$U (a slight decrease of $\sim$1\% in the
 energy range $\sim$400~keV - 1~MeV), while the INL adjustment shows a slight decrease of \nub. CEA variance is too
pessimistic and INL/CEA difference outside the scope INL/CEA relative difference.
\end{itemize}

\begin{figure}[bth]
\includegraphics*[width=\columnwidth, keepaspectratio]{figure21.png}
\caption{INL $^{238}$U inelastic scattering cross section adjustments.}
\label{figure21}
\end{figure}
\begin{figure}[bth]
\includegraphics*[width=\columnwidth, keepaspectratio]{figure22.png}
\caption{CEA $^{238}$U inelastic scattering cross section adjustments.}
\label{figure22}
\end{figure}
\begin{figure}[bth]
\includegraphics*[width=\columnwidth, keepaspectratio]{figure23.png}
\caption{JAEA $^{238}$U inelastic scattering cross section adjustments.}
\label{figure23}
\end{figure}

(d) $^{235}$U capture (Fig.~\ref{figure24}).

The uncertainty values on this parameter are significantly different among the different covariance data
files (see section~\ref{Covariance comparison}).

The larger uncertainties in COMMARA-2.0 (and in CEA covariance data) allow a significant decrease of the
cross section, essentially to improve the \kef\ under prediction of INL. Finally, the JAEA adjustment is a decrease of
the capture cross section only at very high energy where the J-4.0 uncertainties are higher. Elsewhere, the low
uncertainty data in J-4.0 do not allow any significant decrease of the $^{235}$U capture data.

It is worth noting that most teams (INL/ANL, CEA, IRSN and JAEA) point out the necessity of decreasing the $^{235}$U capture
cross section. An equivalent conclusion was given by WPEC Subgroup 29.

\begin{figure*}[tbh]
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure24a.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure24b.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure24c.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure24d.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure24e.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure24f.png}
\caption{Adjustment of $^{235}$U neutron capture cross section obtained by INL (top), CEA (middle), JAEA and IRSN (bottom).}
\label{figure24}
\end{figure*}


(e) For other data, \eg\ $^{23}$Na inelastic and elastic, $^{56}$Fe inelastic scattering cross sections, the adjustments are rather
small and it is rather difficult to extract clear common trends.

In summary, the analysis of these examples suggests that:

\begin{itemize}
\item Adjustment should include all significant parameters in order to provide meaningful indication (see case
 of inelastic scattering and prompt fission spectrum of $^{239}$Pu), and a wide range of integral experiments with different sensitivity profiles.
\item Very different covariance data give rise to different adjustments (case of $^{238}$U(n,n') and $^{235}$U capture). This
point reinforces the need to produce very reliable covariance data and to understand the impact of very small variance data
and of correlations (in energy, among reactions, \etc).
\end{itemize}

\subsection{Impact of Replacing Covariance Data on the Adjustment (Phase II).}

In principle, the central cross section values and the corresponding covariance data of a library must be consistent.
However, here we intentionally replace the covariance data used in the adjustment procedure, though we use
the same values of other adjustment parameters, especially the $C/E$ values which completely depend on the central
cross sections, to analyze the pure effects of the different covariance data to the adjusted results. Two adjustment
cases are surveyed as follows:

\begin{itemize}
\item Case J (Phase I): This is the reference adjustment case. Cross sections and covariance data are both based on the
 JENDL-4.0 library (J-4.0, Refs. 19-20).
\item Case B (Phase II): The covariance data are taken from C-2.0 and no J-4.0 covariance data are supplemented. The other
adjustment parameters are identical with Case J.
\end{itemize}

To investigate the effect of different covariance data, the cases of criticality and the sodium void reactivity have been
investigated. Here we will focus on the case of criticality (\kef).

Fig.~\ref{figure25} compares the \kef\ $C/E$ changes of the two cases due to the adjustment. It is found that the adjusted
$C/E$ values of the two cases are almost identical for \kef\ of small through large cores. In detail, however, there are
some differences between Case J and Case B for the \kef\ of the JOYO Mk-I core which contains the $^{235}$U fuel as well
as plutonium, while the other cores do not include $^{235}$U in fuel. The use of C-2.0 shows better improvement of
JOYO $C/E$ values than J-4.0.

The contribution of $^{235}$U capture cross section adjustment in case B (Phase II) is significantly different by
$\sim$400~pcm between the J-4.0 and C-2.0 covariance data, while those of $^{23}$Na and $^{56}$Fe elastic scattering
are smaller ($\sim$70~pcm each) but opposite in sign. The overall \kef\ correction is of the order of $\sim$200~pcm and
makes the $C/E$ agreement much better.

As for the most important contribution, \ie, $^{235}$U capture, the difference of STDs between C-2.0 and J-4.0 significantly
affected the changing rate of the cross section. The small STD of J-4.0 must constrain the alteration of the
cross section by the adjustment. The large STD of C-2.0 allows the large changing rate of the $^{235}$U capture cross sections
to improve the $C/E$ value of JOYO \kef\ by the adjustment.

\begin{figure*}[!htb]
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure25a.png}
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure25b.png}
\caption{Modifications of $C/E$ of \kef\ values due to adjustment.}
\label{figure25}
\end{figure*}

One can see in Fig.~\ref{figure25} that the initial a priori covariance \textit{does not seem to influence the
final $C/E$ values} as well as their uncertainties. 

This final point is due to the form of generalized least square equations. Let us remind the form of the equation for
a posteriori covariances
\begin{equation} \label{Eq15}
M_{\sigma}'=M_{\sigma}-M_{\sigma}S^{t} \left(M_E + S M_{\sigma} S^{t} \right)^{-1} S M_{\sigma}.
\end{equation}

If $S M_{\sigma} S^{t} >> M_{E}=V_{e}+V_{m}$, then the posterior uncertainties on experiments due to the new
cross section data, $S M_{\sigma}' S^{t}$, is almost equal to $M_E$. $V_e$ stands for integral experiments
covariances and $V_m$ for modeling covariances.

\subsection{Effect of Uncertainties and Correlations (prior/posterior) on ``Target System'' Uncertainties (Phase III)}

In order to investigate the impact of adjustments and in particular of a posteriori correlations, two ``Target Systems'' have
been defined on which the effect of the adjustment is tested. This corresponds to what is expected to be done in practice:
one wants to verify the impact of an adjustment in terms of reduction of uncertainties. Two target systems have been
defined: the JAEA Fast Breeder Reactor (FBR) defined in Ref~\cite{Hayashi_1992} and the ANL Advanced Burner Reactor (ABR),
described in Ref.~\cite{Hoffman_2007}. For this last system, a metal and oxide core fuel versions have been considered,
and for the oxide core, also a recycled fuel core version has been considered.

We evaluated the impact of correlations before and after adjustment on the target systems \kef\ uncertainties. Only the
COMAC-V0 covariance matrices were used. The following uncertainty propagation calculations were performed:

\begin{itemize}
\item Case 1: Full prior/posterior covariance matrix.
\item Case 2: Remove, from previous case, correlations between different isotopes (\eg, no correlation between $^{235}$U and $^{238}$U).
\item Case 3: Remove, from previous case, correlations on reactions for each isotopes (\eg, inelastic and elastic scattering for $^{238}$U).
\item Case 4: Keep only variances.
\end{itemize}

The results are shown in Table~\ref{tab:kunc}.

\begin{table}[tb]
\caption{Impact of correlations on the \kef\ uncertainty of the ``Target Systems.''}
\label{tab:kunc} 
\begin{tabular}{c|c|c} \hline\hline 
ABR Oxide          & Prior covariances   & Posterior covariances  \\ \hline 
Case 1             & 1550 pcm          & 170 pcm               \\
Case 2             & 1550 pcm          & 510 pcm               \\
Case 3             & 1720 pcm          & 680 pcm               \\
Case 4             & 1200 pcm          & 820 pcm               \\ \hline 
JAEA FBR           & Prior covariances   & Posterior covariances   \\ \hline 
Case 1             & 1310 pcm          & 220 pcm               \\
Case 2             & 1310 pcm          & 490 pcm               \\
Case 3             & 1560 pcm          & 610 pcm               \\
Case 4             & 1170 pcm          & 900 pcm               \\ \hline 
ABR Metal          & Prior covariances   & Posterior covariances   \\ \hline 
Case 1             & 1740 pcm          & 250 pcm               \\
Case 2             & 1740 pcm          & 560 pcm               \\
Case 3             & 2020 pcm          & 730 pcm               \\
Case 4             & 1290 pcm          & 850 pcm               \\ \hline 
ABR Oxide Recycled   & Prior covariances  & Posterior covariances  \\ \hline
Case 1             & 1250 pcm          & 260 pcm               \\
Case 2             & 1250 pcm          & 490 pcm               \\
Case 3             & 1400 pcm          & 590 pcm               \\
Case 4             & 1080 pcm          & 820 pcm               \\ \hline\hline
\end{tabular}
\end{table}

For all target systems, the uncertainty using prior and posterior covariance matrices converge to rather similar values
when correlations are deleted. Moreover, one can see that: 

\begin{itemize}
\item Correlations among isotopes (after adjustment) are important and contribute to the uncertainty reduction by
   a factor 2-3 in terms of target system uncertainties.
\item Correlations among reactions for each isotope are significant, as well as correlation between energy groups for each reaction.
\item As for the prior covariance matrix, taking into account cross-correlations (between reactions and energy groups)
 tends to give an overall uncertainty of the same order of magnitude (Case 4 $\rightarrow$ Case 1), except for ABR Metal
 core. But, the effect of energy correlations is an increase of the uncertainty (Case 4 $\rightarrow$ Case 3) and the effect
 of constraint on reactions tends to lower this uncertainty (Case 3 $\rightarrow$ Case 2 $=$ Case 1).
\item As for the posterior covariance matrix, the introduction of correlations gives always a reduction of uncertainty
 Case 4 $\rightarrow$ Case 3 $\rightarrow$ Case 2 $\rightarrow$ Case 1.
\end{itemize}

More results related to the impact of the adjustments on the ``Target Systems'' can be found in~\cite{SG33_2013}. These results do
confirm the trends and the conclusions given above.

\subsection{Calculation of Cook Distance: Influence of Experiments}
\label{Cook distance}

If the adjustment is done by discarding a chosen integral data point I, the results (adjusted cross sections) are
noted: $\left\{{\sigma}^I_p\right\}$. The original adjustment is noted $\left\{{\sigma}_p\right\}$. The Cook distance
is calculated using
\begin{equation} \label{Eq16}
D_I={\left({\sigma}^I_p-{\sigma}_p\right)}^t{{\rm (}M_{\sigma}'{\rm)}}^{-1}\left({\sigma}^I_p-{\sigma}_p\right).
\end{equation}

Low values of $D$ shows a negligible impact of the remote data points. High values of $D$ indicate very influential
experiments. This last point may be due to very low experimental uncertainties as well as very important influence
on final multigroup cross sections.

The calculation of Cook distance was carried out using COMAC-V0 covariances. The results (blue bars) can be seen on
Fig.~\ref{figure26}. The influence of each experiment is detailed by type:

\begin{itemize}
\item  \kef: JOYO is an important experiment in the adjustment for two reasons: all isotopes have an impact on the
 reactivity of the core ($^{235}$U + $^{239}$Pu fuel) and the experimental uncertainty is rather small (180~pcm). If
 we change manually this specific uncertainty, from 180~pcm to 500~pcm, we obtain the red bars on Fig.~\ref{figure26},
 where the weight of JOYO is reduced by almost a factor 10 ; Jezebel $^{240}$Pu and $^{239}$Pu are the second and third
 experiments with the most leverage mostly because of the low experimental uncertainty; Flattop is 4 times less important
 than Jezebel $^{239}$Pu because of its large experimental uncertainty (300~pcm instead of 200~pcm); ZPPR-9 has the same
 influence as Jezebel $^{239}$Pu, thanks to its very small experimental uncertainty (118~pcm); ZPR6-7 and ZPR6-7 High
 $^{240}$Pu content have a negligible impact, in comparison with ZPPR-9, mostly due to their experimental uncertainties ($\sim$230~pcm).
\item  Sodium void reactivity: these experiments seem to be important (the same order of magnitude with Jezebel $^{240}$Pu and JOYO).
 Structural material such as $^{23}$Na and $^{56}$Fe are more sensitive to this kind of experiment than to multiplication
 factor, which can explain the weight of sodium void in the adjustment. Also, the experimental uncertainties for ZPPR-9 Step 3
 and Step 5 are quite low.
\item  Reaction rate ratios: for all cases, this type of integral experiment has a very limited impact in the adjustment
 because the sensitivities are mostly focused on only two reactions in the whole set of cross section parameters.
\end{itemize}

\begin{figure}[!htb]
\includegraphics*[width=0.95\columnwidth, keepaspectratio]{figure26.png}
\caption{Cook distance of experiments. The largest distances indicate the most influential integral parameters.}
\label{figure26}
\end{figure}

\subsection{Stress Tests on the Adjustments}

\subsubsection{Stress test specification}

In the previous paragraphs we have described the cross section adjustment exercise based on the selected 20 fast reactor
experiment benchmarks as the reference case. As seen in the previous discussion, the adjusted results of the reference
case are quite satisfactory from both viewpoints of the integral and differential data. The objective of this section is
to study the impact on the adjusted results when an integral experiment with different nature from the standard 20
experiments set is added to the reference case. This exercise was characterized as the ``Stress test'', \ie, one specific
integral experiment was added with a $C/E$ value very different from 1 to 20 reference integral
experiments, to find what would be the feedback on the adjusted results. 

A larger number of cases are described in detail in Ref.~\cite{SG33_2013}, however here we will summarize only a typical result.

In particular, the following two adjustment cases have been analyzed:

1) Case J4: This is the reference adjustment case with the standard 20 integral data. Cross sections and covariance data are
based on the JENDL-4.0 library (Refs.19-20).

2) ``Stress test'': One integral experiment, \ie, the \kef\ of the ZPR-9/34 core~\cite{Schaefer_2011}, is added to the
reference Case J4. The unique features of the ZPR-9/34 core are:

\begin{itemize}
\item the core region consists of 93\% enriched-uranium and iron, 
\item the height and diameter of the core are 1.8m and 1.2m, respectively,
\item the core is surrounded by stainless steel reflector.
\end{itemize}

The experimental \kef\ value and associated uncertainty of the stress test experiment is based on the ICSBEP
handbook~\cite{ICSBEP_2007}. The \kef\ value is calculated by a continuous-energy Monte Carlo code with
two-dimensional homogenized (r,z) benchmark models, and applied with the corrective factors between the simplified RZ
model and the as-built three-dimensional heterogeneous model which are supplied in the ICSBEP handbook.
The modeling uncertainty is based on the uncertainty estimation associated with the ``Monte Carlo transformation of model''
correction factors to convert the simplified (r,z) model values to the as-built model which is also supplied in the
ICSBEP handbook~\cite{ICSBEP_2007}. The Monte Carlo statistical uncertainties of the simplified (r,z) model calculation are
also added to the modeling uncertainties, though they are negligible compared with the model transformation uncertainties.

\subsubsection{Results of the stress test}

As for the initial results, it is interesting to note that the \kef\ value of ZPR-9/34 is overestimated by +1420~pcm; on the
other hand, \kef\ values of six cores treated in Case J4, numbers 1, 5, 6, 9, 13 and 20, are within only $\pm$530~pcm. The
objective of the ``Stress test'' is to verify whether the adjustment operation could manage the $C/E$ value of ZPR-9/34 very
different from 1 without harmful influence to other integral data and/or cross sections adjustment performances. 

The $C/E$ value changes for all integral data in Case J4 and ``Stress test'' are summarized in Tables~\ref{tab:adj_res} and \ref{tab:strs_res}.
The ``Stress test'' does not seem to give any critical harm to the standard 20 integral data of Case J4. In fact in the
``Stress test'' case, the adjusted results are practically as good as in Case J4. Even the $C/E$ value of the extra
ZPR-9/34 \kef\ which is newly added to the adjustment, changes to almost 1.00.

In both cases the $\chi^2$ test gives an excellent indication of reliability of the adjustment (0.53 and 0.63, respectively).
Moreover, the analysis of the impact of newly added experiment can be analyzed in terms of Eqs.~\eqref{Eq13} and \eqref{Eq14}.
As for the diagnostics using Eq.~\eqref{Eq14}, the values associated to each experiment in the
case of JAEA, here chosen as example, are given in the last column of Tables~\ref{tab:adj_res} and \ref{tab:strs_res}. The ZPR-9/34
\kef\ experiment shows an acceptable value (\ie, $\chi=1.21$ which is below 3 sigma). If we use the parameters defined in section V and based
on Eq.~\eqref{Eq13}, one obtains for the adjustment margin, $AM=-0.005$, the experiment merit, $EM=-1.16$, and the theoretical adjustment margin,
$TAM=-0.17$. These values can be interpreted as follows: the new experiment will marginally contribute to the overall adjustment (\ie, $AM \sim 0$);
however there is merit in the newly added information ($EM < 0$); but the negative value of \textit{TAM} indicates that nuclear data, important for
that specific experiment, have associated uncertainties that are probably underestimated.

\begin{table*}[bth]
\centering
\caption{Results of the adjustment based on JENDL-4.0 (Case J4: Standard 20 integral data) -- reduced $\chi^2=0.53$.}
\label{tab:adj_res}
\begin{minipage}[h]{0.9\textwidth}
\begin{tabular}{c|c|c|c|c|c|c|c|c|c} \hline \hline
\multirow{4}{*}{No.} & \multirow{4}{*}{Core} & & \multicolumn{2}{c|}{\multirow{2}{*}{$C/E$ Value}}
& \multicolumn{2}{c|}{Integral parameter} & \multicolumn{2}{c|}{Nuclear-data-induced} & Ratio of \\
& & Integral & \multicolumn{2}{c|}{ }& \multicolumn{2}{c|}{uncertainty (\%)} & \multicolumn{2}{c|}{uncertainty (\%)} & $|C/E-1|$ \\ \cline{4-9}
& & parameter & \multirow{2}{*}{Before} & \multirow{2}{*}{After} & Experiment &   Modeling   & Before & After & to prior total \\
& & & & & $(\sqrt{V_e})$ & $(\sqrt{V_m})$ & $(\sqrt{SM_{\sigma}S^t})$ & $(\sqrt{SM_{\sigma}'S^t})$ & uncertainty\footnote{Ratio of $|C/E-1|$ to prior total-uncertainty: $|C/E-1|/\sqrt{SM_{\sigma}S^t+V_e+V_m}$.} \\ \hline
  1 & JEZEBEL239 & \kef  &   0.9987  &   0.9997  &     0.20     &     0.03    &    0.69      &    0.15      & 0.18      \\
  2 & JEZEBEL239 & F28/F25        &   0.969   &   0.990   &     1.1      &     0.94    &    3.20      &    1.02      & 0.89      \\
  3 & JEZEBEL239 & F49/F25        &   0.984   &   0.987   &     0.9      &     0.75    &    0.63      &    0.47      & 1.23      \\
  4 & JEZEBEL239 & F37/F25        &   0.979   &   0.989   &     1.4      &     0.80    &    1.50      &    0.67      & 0.93      \\
  5 & JEZEBEL240 & \kef  &   0.9984  &   1.0001  &     0.20     &     0.03    &    0.65      &    0.14      & 0.24      \\
  6 & FLATTOP-PU & \kef  &   0.9986  &   1.0002  &     0.30     &     0.03    &    1.26      &    0.28      & 0.11      \\
  7 & FLATTOP-PU & F28/F25        &   0.977   &   0.998   &     1.1      &     0.84    &    2.94      &    0.97      & 0.70      \\
  8 & FLATTOP-PU & F37/F25        &   0.993   &   1.001   &     1.4      &     0.69    &    1.44      &    0.72      & 0.35      \\
  9 & ZPR6-7     & \kef  &   1.0053  &   1.0029  &     0.23     &     0.03    &    0.82      &    0.12      & 0.62      \\
 10 & ZPR6-7     & F28/F25        &   1.034   &   1.029   &     3.0      &     2.24    &    4.82      &    1.85      & 0.55      \\
 11 & ZPR6-7     & F49/F25        &   0.979   &   0.976   &     2.1      &     1.43    &    1.15      &    0.83      & 0.75      \\
 12 & ZPR6-7     & C28/F25        &   1.017   &   1.011   &     2.4      &     1.22    &    2.00      &    1.12      & 0.50      \\
 13 & ZPR6-7 240 & \kef  &   1.0033  &   1.0010  &     0.22     &     0.03    &    0.81      &    0.12      & 0.39      \\
 14 & ZPPR-9     & \kef  &   1.0021  &   1.0001  &     0.117    &     0.02    &    0.90      &    0.11      & 0.23      \\
 15 & ZPPR-9     & F28/F25        &   0.983   &   0.977   &     2.7      &     2.09    &    5.28      &    2.02      & 0.27      \\
 16 & ZPPR-9     & F49/F25        &   0.999   &   0.996   &     2.0      &     1.21    &    1.15      &    0.83      & 0.03      \\
 17 & ZPPR-9     & C28/F25        &   1.019   &   1.013   &     1.9      &     1.39    &    2.03      &    1.12      & 0.60      \\
 18 & ZPPR-9     & Na Void Step 3 &   1.068   &   1.038   &     1.9      &     5.26    &    5.95      &    3.32      & 0.84      \\
 19 & ZPPR-9     & Na Void Step 5 &   1.052   &   1.014   &     1.9      &     4.96    &    7.31      &    4.04      & 0.58      \\
 20 & JOYO MK-I  & \kef  &   0.9982  &   0.9990  &     0.18     &     0.03    &    0.58      &    0.16      & 0.29      \\ \hline \hline 
\end{tabular}
\end{minipage}
\end{table*}

\begin{table*}[bth]
\centering
\caption{Results of the ``stress test'' adjustment with ZPR-9/34 added to the 20 integral data, (Case J4) -- reduced $\chi^2=0.63$.}
\label{tab:strs_res}
\begin{minipage}[h]{0.9\textwidth}
\begin{tabular}{c|c|c|c|c|c|c|c|c|c} \hline \hline
\multirow{4}{*}{No.} & \multirow{4}{*}{Core} & & \multicolumn{2}{c|}{\multirow{2}{*}{$C/E$ Value}}
& \multicolumn{2}{c|}{Integral parameter} & \multicolumn{2}{c|}{Nuclear-data-induced} & Ratio of \\
& & Integral & \multicolumn{2}{c|}{ }& \multicolumn{2}{c|}{uncertainty (\%)} & \multicolumn{2}{c|}{uncertainty (\%)} & $|C/E-1|$ \\ \cline{4-9}
& & parameter & \multirow{2}{*}{Before} & \multirow{2}{*}{After} & Experiment &   Modeling   & Before & After & to prior total \\
& & & & & $(\sqrt{V_e})$ & $(\sqrt{V_m})$ & $(\sqrt{SM_{\sigma}S^t})$ & $(\sqrt{SM_{\sigma}'S^t})$ & uncertainty\footnote{Ratio of $|C/E-1|$ to prior total-uncertainty: $|C/E-1|/\sqrt{SM_{\sigma}S^t+V_e+V_m}$.} \\ \hline
  1 & JEZEBEL239 & \kef  &   0.9987  &   0.9997  &     0.20     &     0.03    &    0.69      &    0.15      & 0.18      \\
  2 & JEZEBEL239 & F28/F25        &   0.969   &   0.989   &     1.1      &     0.94    &    3.20      &    1.02      & 0.89      \\
  3 & JEZEBEL239 & F49/F25        &   0.984   &   0.987   &     0.9      &     0.75    &    0.63      &    0.47      & 1.23      \\
  4 & JEZEBEL239 & F37/F25        &   0.979   &   0.989   &     1.4      &     0.80    &    1.50      &    0.67      & 0.93      \\
  5 & JEZEBEL240 & \kef  &   0.9984  &   1.0000  &     0.20     &     0.03    &    0.65      &    0.14      & 0.24      \\
  6 & FLATTOP-PU & \kef  &   0.9986  &   1.0007  &     0.30     &     0.03    &    1.26      &    0.28      & 0.11      \\
  7 & FLATTOP-PU & F28/F25        &   0.977   &   0.997   &     1.1      &     0.84    &    2.94      &    0.97      & 0.70      \\
  8 & FLATTOP-PU & F37/F25        &   0.993   &   1.001   &     1.4      &     0.69    &    1.44      &    0.72      & 0.35      \\
  9 & ZPR6-7     & \kef  &   1.0053  &   1.0028  &     0.23     &     0.03    &    0.82      &    0.12      & 0.62      \\
 10 & ZPR6-7     & F28/F25        &   1.034   &   1.033   &     3.0      &     2.24    &    4.82      &    1.84      & 0.55      \\
 11 & ZPR6-7     & F49/F25        &   0.979   &   0.979   &     2.1      &     1.43    &    1.15      &    0.81      & 0.75      \\
 12 & ZPR6-7     & C28/F25        &   1.017   &   1.011   &     2.4      &     1.22    &    2.00      &    1.12      & 0.50      \\
 13 & ZPR6-7 240 & \kef  &   1.0033  &   1.0009  &     0.22     &     0.03    &    0.81      &    0.12      & 0.39      \\
 14 & ZPPR-9     & \kef  &   1.0021  &   1.0002  &     0.117    &     0.02    &    0.90      &    0.11      & 0.23      \\
 15 & ZPPR-9     & F28/F25        &   0.983   &   0.979   &     2.7      &     2.09    &    5.28      &    2.01      & 0.27      \\
 16 & ZPPR-9     & F49/F25        &   0.999   &   0.999   &     2.0      &     1.21    &    1.15      &    0.82      & 0.03      \\
 17 & ZPPR-9     & C28/F25        &   1.019   &   1.013   &     1.9      &     1.39    &    2.03      &    1.12      & 0.60      \\
 18 & ZPPR-9     & Na Void Step 3 &   1.068   &   1.046   &     1.9      &     5.26    &    5.95      &    3.29      & 0.84      \\
 19 & ZPPR-9     & Na Void Step 5 &   1.052   &   1.019   &     1.9      &     4.96    &    7.31      &    4.03      & 0.58      \\
 20 & JOYO MK-I  & \kef  &   0.9982  &   0.9984  &     0.18     &     0.03    &    0.58      &    0.16      & 0.29      \\
 21 & ZPR-9/34   & \kef  &   1.0142  &   1.0012  &     0.11     &     0.24    &    1.15      &    0.25      & 1.21      \\ \hline \hline 
\end{tabular}
\end{minipage}
\end{table*}

Finally, it is needed to confirm that the adjustment gives no harm not only in terms of integral data a posteriori values,
but also in terms of differential cross sections. For this purpose, the nuclide- and reaction-wise contributions to the total
\kef\ modifications were investigated, comparing the cases with and without stress test 

In the ``Stress test'' case, it is found that the large improvement of the $C/E$ value, approximately -1300~pcm, is attained
by the cross section changes of $^{56}$Fe capture and elastic scattering reactions, which are considered to result in the
negative reactivity of the absorption effect by the iron in the core region, and the neutron leakage enhancement by the
stainless steel reflector. These two cross section adjustments are responsible for $\sim$700 and $\sim$600~pcm, respectively 

Fig.~\ref{figure27} shows the cross section modifications for the $^{56}$Fe capture reaction. In the reference Case J4, there
are small changes of the cross sections. However, quite large cross section alterations occur in the ``Stress test''. The order
of magnitude is +11\% below 100~keV, which is very close to the STD value, $\pm$10\%. Changes of this magnitude might be close
to or exceed the limitation of allowance from the viewpoint of the nuclear data evaluation.

Fig.~\ref{figure28} illustrates the changes of the $^{56}$Fe elastic scattering cross sections. In the Case J4, there are
small alterations. On the other hand, in the ``Stress test'', the $^{56}$Fe elastic cross sections are decreased to adjust\
\kef\ values of the stainless steel or iron reflector core of ZPR-9/34. These cross section changes are quite large, but
still within the STD values, which may be acceptable from the nuclear data viewpoint.

The fact that the required adjustments are so close to the STD values in the JAEA covariance matrix, can be associated to
the negative, even if small value, of the \textit{TAM} parameters, as discussed in the previous paragraph.

As far as the other cross sections, the stress test does not modify significantly the adjustment results obtained in the
case J4. As an example, the adjustments of the $^{239}$Pu capture cross sections are shown in Fig.~\ref{figure29}.
Both in Case J4 and in the ``Stress test'' case, the change is approximately +3-5\%, which is within one standard
deviation (STD) value of the JENDL-4.0 covariance, that is, $\pm$6-9\% in the dominant energy region.

In summary the example shown here and further stress tests reported in~\cite{SG33_2013}, tend to confirm the robustness of
the adjustment procedures.

\begin{figure}[tbh]
 \includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure27.png}
 \caption{Change of cross sections by adjustment: $^{56}$Fe$(n,\gamma)$.}
 \label{figure27}
\end{figure}
\begin{figure}[tbh]
 \includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure28.png}
 \caption{Change of cross sections by adjustment: $^{56}$Fe$(n,n)$.}
 \label{figure28}
\end{figure}
\begin{figure}[tbh]
 \includegraphics*[width=1.0\columnwidth, keepaspectratio]{figure29.png}
 \caption{Change of cross sections by adjustment: $^{239}$Pu$(n,\gamma)$.}
 \label{figure29}
\end{figure}

\subsection{Conclusion of Adjustment Results}

This first analysis indicates that:

\begin{itemize}
\item Adjustment should include all significant parameters in order to provide meaningful indications
 (see case of inelastic scattering and chi of $^{239}$Pu), and a wide range of integral experiments with different sensitivity profiles.
\item Very different covariance data give rise to different adjustments (case of $^{238}$U(n,n') and $^{235}$U capture).
 This point reinforces the need to produce very reliable covariance data and to understand the impact of very small variance
 data and of correlations (in energy, among reactions, \etc).
\item Initial $C/E$'s are driving the path to adjustment in some cases (see example of CEA $^{238}$U(n,n') trends depending
 on $C/E$'s values in~\cite{SG33_2013}). 
\item Experimental uncertainties have to be correctly quantified, because they drive the weight in the adjustment process
 (see Cook distance). In other words, a mis-estimated experimental uncertainty can lead to biased trends on cross sections
 and over-estimation of the uncertainty reduction.
\item Final calculated uncertainties on benchmarks do not seem to depend on chosen a priori cross section covariance;
 uncertainty reduction through integral experiment is driven by integral experiment uncertainties.
\item A posteriori cross section covariances are only driven by the competition between a priori covariance matrix and
 initial experimental matrix with deterministic adjustments procedure.
\end{itemize}

\section{Conclusions}

Subgroup 33 has succeeded in providing a deeper understanding of nuclear data adjustment methods and of their application.

The findings of the Subgroup 33 have pointed out that the statistical adjustments methodologies in use worldwide for different
reactor analysis and design purposes are well understood and that they are essentially equivalent. 

The results of the adjustments indicate, for some important data, common trends for modification even if starting from different
basic nuclear data and different covariance matrices. The results obtained show also some degree of robustness in the sense that
the observed trends can ``survive'' rather severe ``stress tests''. 

In this respect, these methodologies can provide a powerful tool for nuclear data (and associated uncertainties) improvement if
used in an appropriate manner. In fact, it has been indicated that the associated sensitivity analysis requires careful use
of existing methods and that the choice of specific integral experiments of different types (critical masses but also reaction
rates, reactivity coefficients and irradiation experiments) and sensitive to different energy neutron spectra, is of high
relevance to avoid as much as possible compensating effects in the adjustments. 

Finally, it has been pointed out the crucial role of the covariance data used, both those associated to the nuclear data and those
associated to the integral experiments. The a posteriori correlations are mainly responsible for the uncertainty reduction of
parameters of reference design systems. Their physics meaning and appropriate utilization will need further study.

The deeper understanding of the methodologies and of their applications implies that the role for cross section adjustment is more
and more perceived as that of providing useful feedback to evaluators and differential measurement experimentalists in order to
improve the knowledge of neutron cross sections to be used in a wider range of applications.

This new role for cross section adjustment requires tackling and solving a new series of issues: definition of criteria to assess
the reliability and robustness of an adjustment; requisites to assure the quantitative validity of the covariance data; criteria
to alert for inconsistency between differential and integral data; definition of consistent approaches to use both adjusted data
and a posteriori covariance data to improve quantitatively nuclear data files; provide methods and define conditions to generalize
the results of an adjustment in order to evaluate the ``extrapolability'' of the results of an adjustment to a different range of
applications (\eg, different reactor systems) for which the adjustment was not initially intended; suggest guidelines to enlarge
the experimental data base in order to meet needs that were identified by the cross section adjustment.

\vspace{3mm}
\centerline{\bf Acknowledgements}
\vspace{3mm}

The authors wish to express their sincere gratitude to the staff of the National Nuclear Data Center, especially S.~Hoblit who
took care of \LaTeX\ corrections in the course of the review process. The work at INL is supported by the U.S. Department of Energy,
Office of Nuclear Energy, under DOE Idaho Operations Office Contract DE-AC07-05ID14517.

%%% IMPORTANT: When preparing bibliography observe strictly the layout below (initials in front of the names,
%%% no names in capitals, not more than four authors, no titles, volume number in bold, year at the end within parenthesis,
%%% dot (.) at the end.

\begin{thebibliography}{9}

\bibitem{Aliberti_2004} G.~Aliberti \etal, ``Impact of Nuclear Data Uncertainties on Transmutation of Actinides in Accelerator-Driven
Assemblies,'' {\sc Nucl. Sci. Eng.} {\bf 46}, 13-50 (2004).

\bibitem{Aliberti_2006} G.~Aliberti \etal, ``Nuclear Data Sensitivity, Uncertainty and Target Accuracy Assessment for Future
Nuclear Systems'', {\sc Ann. of Nucl. Energy} {\bf 33}, 700-733 (2006).

\bibitem{SG26_2008} ``Uncertainty and Target Accuracy Assessment for Innovative Systems Using Recent Covariance Data
Evaluations'', {\sc International Evaluation Co-operation} {\bf 26}, NEA/WPEC-26, OECD/NEA, Paris (2008).

\bibitem{Cecchini_1964} G.~Cecchini, U.~Farinelli, A.~Gandini and M.~Salvatores, ``Analysis of integral data for few-group parameter
evaluation of fast reactors'', Proc. 3rd Int. Conf. Peaceful Uses Atomic Energy, Geneva, P/627, pp.388-397 (1964).

\bibitem{Mitani_1972} H.~Mitani and H.~Kuroi, ``Adjustment of Group Cross Sections by Means of Integral
Data, (I), (II)'', {\sc J. Nucl. Sci. and Tech.} {\bf 9}, pp.383-394 and p.642-657 (1972).

\bibitem{Chaudat_1973} J.P.~Chaudat, J.Y.~Barre and A.~Khairallah, ``Improvements of the Predicted Characteristics for
Fast Power Reactor from Integral Experiments: Cadarache Version III Multigroup Cross Section Set'', Proc. of the Symp.
Physics of Fast Reactor, Tokyo, Vol. 3, pp.1207 (1973).

\bibitem{SG33_2011} ``Assessment of Existing Nuclear Data Adjustment Methodologies'', International Evaluation Co-operation,
Intermediate Report of WPEC Subgroup 33, NEA/NSC/WPEC/DOC(2010)429, OECD/NEA, Paris, 2011.

\bibitem{ICSBEP_2007} International Handbook of Evaluated Criticality Safety Benchmark Experiments, NEA/NSC/DOC(95)03, September 2007 Edition.

\bibitem{IRPhE_2007} International Handbook of Evaluated Reactor Physics Benchmark Experiments, NEA/NSC/DOC(2006)1, March 2007 Edition.

\bibitem{LeSage_2001} L.G.~LeSage, ``An Overview of the Argonne National Laboratory Fast Critical Experiments, 1963-1990'', ANL-NT-175, April 2001.

\bibitem{Usachev_1967} L.N.~Usachev, Proc. Int. Conf. on Peaceful Uses of Nuclear Energy, Vol. 5, 503, Geneva (1955) and A.
Gandini: ``A Generalized Perturbation Method for Bi-linear Functionals of the Real and Adjoint Neutron Fluxes'', {\sc J.
of Nucl. Energy} {\bf 21}, pp.755-765 (1967).

\bibitem{Ishikawa_2005} M.~Ishikawa, ``Recent application of nuclear data to fast reactor core analysis and design in Japan'',
International Conference on Nuclear Data for Science and Technology, Santa Fe, Sept 26 - Oct 1, 2004, p. 1405 (2005).

\bibitem{Rimpault_2002} G.~Rimpault \etal, ``The ERANOS data and code system for fast reactor neutronic analyses'', Proceedings
of the International Conference on the New Frontier of Nuclear Technology: Reactor Physics, Safety and High-Performance
Computing (PHYSOR 2002), Seoul, Korea (2002).

\bibitem{Kodeli_2001} I.~Kodeli, ``Multidimensional Deterministic Nuclear Data Sensitivity and Uncertainty Code System:
Method and Application'', {\sc Nucl. Sci. Eng.} {\bf 138}, 45-66 (2001). 

\bibitem{Alcouffe_1995} R.E.~Alcouffe \etal,``DANTSYS: A Diffusion Accelerated Neutral Particle Transport Code System'', Los Alamos
National Laboratory report, LA-12969-M (1995).

\bibitem{SCALE_2009} ``TSUNAMI-3D: Control module for three-dimensional cross-section sensitivity and uncertainty analysis
for criticality (SCALE, VOL I, Sect. C9). Technical Report ORNL/TM-2005/39, Oak Ridge National Laboratory (2009).

\bibitem{Gandini_1986} A.~Gandini, G.~Palmiotti and M.~Salvatores, ``Equivalent Generalized Perturbation Theory
(EGPT)'', {\sc Ann. Nucl. Energy} {\bf 13}, 109 (1986).

\bibitem{Smith_1991} D.L.~Smith, ``Probability, Statistics, and Data Uncertainties in Nuclear Science and Technology'',
An OECD Nuclear Energy Agency Nuclear Data Committee Series, Nuclear Physics and Nuclear Data in Science and Technology, Volume 4 (1991). 

\bibitem{Shibata_2011} K.~Shibata \etal, ``JENDL-4.0: A New Library for Nuclear Science and Engineering'',
{\sc J. Nucl. Sci. and Tech.} {\bf 48}, 1-30 (2011), and JENDL-4.0 homepage, http://wwwndc.jaea.go.jp/jendl/j40/j40.html.

\bibitem{Iwamoto_2011} O.~Iwamoto, T.~Nakagawa and S.~Chiba: ``Covariance Evaluation for Actinide Nuclear
Data in JENDL-4.0'', {\sc J. of Korean Phys. Soc.} {\bf 59}, 1224-1229 (2011). 

\bibitem{Herman_2011a} M.~Herman \etal, ``COMMARA-2.0 Neutron Cross Section Covariance Library'', BNL-94830-2011, U.S. Department of Energy (March 2011). 

\bibitem{Chadwick_2006} M.B.~Chadwick \etal, ``ENDF/B-VII.0: Next Generation Evaluated Nuclear Data Library for Nuclear
Science and Technology'', {\sc Nuclear Data Sheets} {\bf 107}, 2931-3060 (2006). 

\bibitem{DeSaintJean_2011} C.~De~Saint~Jean \etal, ``Uncertainty Evaluation of Nuclear Reaction Model Parameters
Using Integral and Microscopic Measurements with the CONRAD Code'', {\sc J. of Korean Phys. Soc.} {\bf 59}, 1276-1279 (2011). 

\bibitem{SG33_2013} Final report of WPEC Subgroup 33, {\sc International Evaluation Co-operation} {\bf 33},
NEA/WPEC-33, OECD/NEA, Paris (to be published).

\bibitem{Poenitz_1981} W.P.~Poenitz, ``Data Interpretation, Objective Evaluation Procedures and Mathematical
Techniques for the Evaluation of Energy-dependent Ratio, Shape and Cross Section Data'', Proceedings of the Conference
on Nuclear Data Evaluation Methods and Procedures, Brookhaven National Laboratory, New York, Sep. 22-25, 1980, BNL-NCS-51363, pp.249-289 (1981). 

\bibitem{Muir_1999} D.W.~Muir, ``ZOTT99, Data Evaluation Using Partitioned Least-Squares'', Code package IAEA1371/01, NEA
Computer Program Service (1999). 

\bibitem{Kawano_1997} T.~Kawano and K.~Shibata, ``Covariance Evaluation System'', Japan Atomic Energy Research
Institute, JAERI-Data/Code 97-037 (September 1997).

\bibitem{Herman_2011b} M.~Herman, ``Development of ENDF/B-VII.1 and Its Covariance Component'', {\sc J.
of Korean Phys. Soc.} {\bf 59}, 1034-1039 (2011).

\bibitem{Oblozinsky_2010} P.~Oblo{\v z}insk{\'y}, Y.-S.~Cho, C.M.~Mattoon and S.F.~Mughabghab, ``Formalism for neutron cross section
covariances in the resonance region using kernel approximation'', Brookhaven National Laboratory, BNL-91287-2010 (April 2010). 

\bibitem{Little_2008} R.C.~Little \etal, ``Low-fidelity Covariance Project'', {\sc Nuclear Data Sheets} {\bf 109}, 2828-2833 (2008).

\bibitem{Mughabghab_2006} S.F.~Mughabghab, ``Atlas of Neutron Resonances, Resonance Parameters and Thermal Cross
Sections, Z = 1-100'', 5th Edition, Elsevier, Amsterdam (2006).

\bibitem{Koning_2007} A.J.~Koning, ``New working methods for nuclear data evaluation: how to make a nuclear
data library?'', Proceedings of International Conference on Nuclear Data for Science and Technology, ND2007, DOI: 10.1051/ndata:07683 (2007). 

\bibitem{Rochman_2011} D.~Rochman \etal, ``Nuclear Data Uncertainty Propagation: Total Monte Carlo vs.
Covariances'', {\sc J. of Korean Phys. Soc.} {\bf 59}, 1236-1241 (2011). 

\bibitem{Koning_2010} A.J.~Koning and D.~Rochman, http://www.talys.eu/tendl-2010 (2010).

\bibitem{Rearden_2011} B.T.~Rearden \etal, ``Sensitivity and Uncertainty Analysis Capabilities and
Data in SCALE'', {\sc Nucl. Technol.} {\bf 174}, 236-288 (2011).

\bibitem{Dupont_2011} E.~Dupont, A.J.~Koning and N.~Otuka, ``Exploratory Data Analysis of the EXFOR
Database'', {\sc J. of Korean Phys. Soc.} {\bf 59}, 1333-1336 (2011).

\bibitem{Ikegami_2011} T.~Ikegami, ``ZPPR-9 Experiment: A 650 MWe-class Sodium-cooled MOX-fueled FBR Core Mock-up
Critical Experiment with Clean Core of Two Homogeneous Zones'', International Handbook of Evaluated Reactor Physics
Benchmark Experiments (IRPhE), ZPPR-LMFR-EXP-002, NEA/NSC/DOC(2006)1, OECD/NEA, 2011.

\bibitem{Lell_2011} R.M.~Lell, J.A.~Morman, R.W.~Schaefer and R.D.~McKnight, ``ZPR-6 Assembly 7 Experiments: A Fast
Reactor Core with Mixed (Pu,U)-Oxide Fuel and Sodium with a Thick Depleted Uranium Reflector'', International
Handbook of Evaluated Reactor Physics Benchmark Experiments (IRPhE), ZPR-LMFR-EXP-001, NEA/NSC/DOC(2006)1, OECD/NEA, 2011.

\bibitem{Schaefer_2011} R.W.~Schaefer, K.A.~Bunde and P.J.~Collins, ``The Uranium/Iron Benchmark Assembly: A 235U(93\%)/Iron
Cylinder Reflected by Stainless Steel'', International Handbook of Evaluated Criticality Safety Benchmark Experiment
(ICSBEP), Volume II, NEA/NSC/DOC(95)03, 2011.

\bibitem{Hayashi_1992} H.~Hayashi \etal, ``Progress Report of the Design Study on a Large Scale Reactor'', Power reactor
and Nuclear fuel development Corporation, PNC TN9410 92-137 (1992).

\bibitem{Hoffman_2007} E.A.~Hoffman, W.S.~Yang, R.N.~Hill, ``A study on variable conversion ratio for fast burner
reactor'' {\sc Trans. Am. Nucl. Soc.} {\bf 96} (2007).

\end{thebibliography}
\end{document}
