diff --git a/beacon-chain/blockchain/BUILD.bazel b/beacon-chain/blockchain/BUILD.bazel index 515518f145..680921681e 100644 --- a/beacon-chain/blockchain/BUILD.bazel +++ b/beacon-chain/blockchain/BUILD.bazel @@ -6,7 +6,6 @@ go_library( "chain_info.go", "info.go", "log.go", - "metrics.go", "process_attestation.go", "process_attestation_helpers.go", "process_block.go", @@ -19,6 +18,7 @@ go_library( visibility = ["//beacon-chain:__subpackages__"], deps = [ "//beacon-chain/blockchain/forkchoice:go_default_library", + "//beacon-chain/blockchain/metrics:go_default_library", "//beacon-chain/cache:go_default_library", "//beacon-chain/cache/depositcache:go_default_library", "//beacon-chain/core/blocks:go_default_library", @@ -46,8 +46,6 @@ go_library( "@com_github_emicklei_dot//:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", "@com_github_pkg_errors//:go_default_library", - "@com_github_prometheus_client_golang//prometheus:go_default_library", - "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", "@com_github_prysmaticlabs_go_ssz//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", diff --git a/beacon-chain/blockchain/forkchoice/BUILD.bazel b/beacon-chain/blockchain/forkchoice/BUILD.bazel index 543c34702d..57c48ef246 100644 --- a/beacon-chain/blockchain/forkchoice/BUILD.bazel +++ b/beacon-chain/blockchain/forkchoice/BUILD.bazel @@ -5,7 +5,6 @@ go_library( srcs = [ "doc.go", "log.go", - "metrics.go", "process_attestation.go", "process_block.go", "service.go", @@ -13,9 +12,9 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/forkchoice", visibility = ["//beacon-chain:__subpackages__"], deps = [ + "//beacon-chain/blockchain/metrics:go_default_library", "//beacon-chain/cache:go_default_library", "//beacon-chain/core/blocks:go_default_library", - "//beacon-chain/core/epoch/precompute:go_default_library", "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/core/state:go_default_library", "//beacon-chain/db:go_default_library", @@ -30,8 +29,6 @@ go_library( "//shared/traceutil:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", "@com_github_pkg_errors//:go_default_library", - "@com_github_prometheus_client_golang//prometheus:go_default_library", - "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", "@com_github_prysmaticlabs_go_ssz//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", diff --git a/beacon-chain/blockchain/forkchoice/metrics.go b/beacon-chain/blockchain/forkchoice/metrics.go deleted file mode 100644 index d81a6ce9b0..0000000000 --- a/beacon-chain/blockchain/forkchoice/metrics.go +++ /dev/null @@ -1,160 +0,0 @@ -package forkchoice - -import ( - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute" - "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" - stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" - "github.com/prysmaticlabs/prysm/shared/bytesutil" - "github.com/prysmaticlabs/prysm/shared/params" -) - -var ( - beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "beacon_finalized_epoch", - Help: "Last finalized epoch of the processed state", - }) - beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "beacon_finalized_root", - Help: "Last finalized root of the processed state", - }) - beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "beacon_current_justified_epoch", - Help: "Current justified epoch of the processed state", - }) - beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "beacon_current_justified_root", - Help: "Current justified root of the processed state", - }) - beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "beacon_previous_justified_epoch", - Help: "Previous justified epoch of the processed state", - }) - beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "beacon_previous_justified_root", - Help: "Previous justified root of the processed state", - }) - sigFailsToVerify = promauto.NewCounter(prometheus.CounterOpts{ - Name: "att_signature_failed_to_verify_with_cache", - Help: "Number of attestation signatures that failed to verify with cache on, but succeeded without cache", - }) - validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "validator_count", - Help: "The total number of validators", - }, []string{"state"}) - validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "validators_total_balance", - Help: "The total balance of validators, in GWei", - }, []string{"state"}) - validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "validators_total_effective_balance", - Help: "The total effective balance of validators, in GWei", - }, []string{"state"}) - currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "current_eth1_data_deposit_count", - Help: "The current eth1 deposit count in the last processed state eth1data field.", - }) - totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "total_eligible_balances", - Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch", - }) - totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "total_voted_target_balances", - Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch", - }) -) - -func reportEpochMetrics(state *stateTrie.BeaconState) { - currentEpoch := helpers.CurrentEpoch(state) - - // Validator instances - pendingInstances := 0 - activeInstances := 0 - slashingInstances := 0 - slashedInstances := 0 - exitingInstances := 0 - exitedInstances := 0 - // Validator balances - pendingBalance := uint64(0) - activeBalance := uint64(0) - activeEffectiveBalance := uint64(0) - exitingBalance := uint64(0) - exitingEffectiveBalance := uint64(0) - slashingBalance := uint64(0) - slashingEffectiveBalance := uint64(0) - - validators := state.Validators() - for i, validator := range validators { - valBalance, err := state.BalanceAtIndex(uint64(i)) - if err != nil { - log.WithError(err).Error("could not get balance for validator") - return - } - if validator.Slashed { - if currentEpoch < validator.ExitEpoch { - slashingInstances++ - slashingBalance += valBalance - slashingEffectiveBalance += validator.EffectiveBalance - } else { - slashedInstances++ - } - continue - } - if validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch { - if currentEpoch < validator.ExitEpoch { - exitingInstances++ - exitingBalance += valBalance - exitingEffectiveBalance += validator.EffectiveBalance - } else { - exitedInstances++ - } - continue - } - if currentEpoch < validator.ActivationEpoch { - pendingInstances++ - pendingBalance += valBalance - continue - } - activeInstances++ - activeBalance += valBalance - activeEffectiveBalance += validator.EffectiveBalance - } - validatorsCount.WithLabelValues("Pending").Set(float64(pendingInstances)) - validatorsCount.WithLabelValues("Active").Set(float64(activeInstances)) - validatorsCount.WithLabelValues("Exiting").Set(float64(exitingInstances)) - validatorsCount.WithLabelValues("Exited").Set(float64(exitedInstances)) - validatorsCount.WithLabelValues("Slashing").Set(float64(slashingInstances)) - validatorsCount.WithLabelValues("Slashed").Set(float64(slashedInstances)) - validatorsBalance.WithLabelValues("Pending").Set(float64(pendingBalance)) - validatorsBalance.WithLabelValues("Active").Set(float64(activeBalance)) - validatorsBalance.WithLabelValues("Exiting").Set(float64(exitingBalance)) - validatorsBalance.WithLabelValues("Slashing").Set(float64(slashingBalance)) - validatorsEffectiveBalance.WithLabelValues("Active").Set(float64(activeEffectiveBalance)) - validatorsEffectiveBalance.WithLabelValues("Exiting").Set(float64(exitingEffectiveBalance)) - validatorsEffectiveBalance.WithLabelValues("Slashing").Set(float64(slashingEffectiveBalance)) - - // Last justified slot - if c := state.CurrentJustifiedCheckpoint(); c != nil { - beaconCurrentJustifiedEpoch.Set(float64(c.Epoch)) - beaconCurrentJustifiedRoot.Set(float64(bytesutil.ToLowInt64(c.Root))) - } - // Last previous justified slot - if c := state.PreviousJustifiedCheckpoint(); c != nil { - beaconPrevJustifiedEpoch.Set(float64(c.Epoch)) - beaconPrevJustifiedRoot.Set(float64(bytesutil.ToLowInt64(c.Root))) - } - // Last finalized slot - if c := state.FinalizedCheckpoint(); c != nil { - beaconFinalizedEpoch.Set(float64(c.Epoch)) - beaconFinalizedRoot.Set(float64(bytesutil.ToLowInt64(c.Root))) - } - if e := state.Eth1Data(); e != nil { - currentEth1DataDepositCount.Set(float64(e.DepositCount)) - } - - if precompute.Balances != nil { - totalEligibleBalances.Set(float64(precompute.Balances.PrevEpoch)) - totalVotedTargetBalances.Set(float64(precompute.Balances.PrevEpochTargetAttesters)) - } -} diff --git a/beacon-chain/blockchain/forkchoice/process_block.go b/beacon-chain/blockchain/forkchoice/process_block.go index 01edd8bd97..1d702f35e6 100644 --- a/beacon-chain/blockchain/forkchoice/process_block.go +++ b/beacon-chain/blockchain/forkchoice/process_block.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-ssz" + "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/metrics" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/state" "github.com/prysmaticlabs/prysm/beacon-chain/db/filters" @@ -131,7 +132,7 @@ func (s *Store) OnBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) (* // Epoch boundary bookkeeping such as logging epoch summaries. if postState.Slot() >= s.nextEpochBoundarySlot { logEpochData(postState) - reportEpochMetrics(postState) + metrics.ReportEpochMetrics(postState) // Update committees cache at epoch boundary slot. if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil { @@ -259,7 +260,7 @@ func (s *Store) OnBlockInitialSyncStateTransition(ctx context.Context, signed *e // Epoch boundary bookkeeping such as logging epoch summaries. if postState.Slot() >= s.nextEpochBoundarySlot { - reportEpochMetrics(postState) + metrics.ReportEpochMetrics(postState) s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState)) } diff --git a/beacon-chain/blockchain/metrics/BUILD.bazel b/beacon-chain/blockchain/metrics/BUILD.bazel new file mode 100644 index 0000000000..713386df63 --- /dev/null +++ b/beacon-chain/blockchain/metrics/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["metrics.go"], + importpath = "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/metrics", + visibility = ["//beacon-chain:__subpackages__"], + deps = [ + "//beacon-chain/core/epoch/precompute:go_default_library", + "//beacon-chain/state:go_default_library", + "//shared/bytesutil:go_default_library", + "//shared/params:go_default_library", + "@com_github_prometheus_client_golang//prometheus:go_default_library", + "@com_github_prometheus_client_golang//prometheus/promauto:go_default_library", + ], +) diff --git a/beacon-chain/blockchain/metrics.go b/beacon-chain/blockchain/metrics/metrics.go similarity index 76% rename from beacon-chain/blockchain/metrics.go rename to beacon-chain/blockchain/metrics/metrics.go index 82bcb65ac4..455b0f6186 100644 --- a/beacon-chain/blockchain/metrics.go +++ b/beacon-chain/blockchain/metrics/metrics.go @@ -1,4 +1,4 @@ -package blockchain +package metrics import ( "github.com/prometheus/client_golang/prometheus" @@ -18,89 +18,75 @@ var ( Name: "beacon_head_slot", Help: "Slot of the head block of the beacon chain", }) - competingBlks = promauto.NewCounter(prometheus.CounterOpts{ + // CompetingBlks is the number of the competing blocks happened over time. + CompetingBlks = promauto.NewCounter(prometheus.CounterOpts{ Name: "competing_blocks", Help: "The # of blocks received and processed from a competing chain", }) - processedBlkNoPubsub = promauto.NewCounter(prometheus.CounterOpts{ - Name: "processed_no_pubsub_block_counter", - Help: "The # of processed block without pubsub, this usually means the blocks from sync", - }) - processedBlkNoPubsubForkchoice = promauto.NewCounter(prometheus.CounterOpts{ - Name: "processed_no_pubsub_forkchoice_block_counter", - Help: "The # of processed block without pubsub and forkchoice, this means indicate blocks from initial sync", - }) - processedBlk = promauto.NewCounter(prometheus.CounterOpts{ - Name: "processed_block_counter", - Help: "The # of total processed in block chain service, with fork choice and pubsub", - }) - processedAttNoPubsub = promauto.NewCounter(prometheus.CounterOpts{ - Name: "processed_no_pubsub_attestation_counter", - Help: "The # of processed attestation without pubsub, this usually means the attestations from sync", - }) headFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_head_finalized_epoch", + Name: "head_finalized_epoch", Help: "Last finalized epoch of the head state", }) headFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_head_finalized_root", + Name: "head_finalized_root", Help: "Last finalized root of the head state", }) beaconFinalizedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_beacon_finalized_epoch", + Name: "beacon_finalized_epoch", Help: "Last finalized epoch of the processed state", }) beaconFinalizedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_beacon_finalized_root", + Name: "beacon_finalized_root", Help: "Last finalized root of the processed state", }) beaconCurrentJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_beacon_current_justified_epoch", + Name: "beacon_current_justified_epoch", Help: "Current justified epoch of the processed state", }) beaconCurrentJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_beacon_current_justified_root", + Name: "beacon_current_justified_root", Help: "Current justified root of the processed state", }) beaconPrevJustifiedEpoch = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_beacon_previous_justified_epoch", + Name: "beacon_previous_justified_epoch", Help: "Previous justified epoch of the processed state", }) beaconPrevJustifiedRoot = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_beacon_previous_justified_root", + Name: "beacon_previous_justified_root", Help: "Previous justified root of the processed state", }) validatorsCount = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "chain_service_validator_count", + Name: "validator_count", Help: "The total number of validators", }, []string{"state"}) validatorsBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "chain_service_validators_total_balance", + Name: "validators_total_balance", Help: "The total balance of validators, in GWei", }, []string{"state"}) validatorsEffectiveBalance = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "chain_service_validators_total_effective_balance", + Name: "validators_total_effective_balance", Help: "The total effective balance of validators, in GWei", }, []string{"state"}) currentEth1DataDepositCount = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_current_eth1_data_deposit_count", + Name: "current_eth1_data_deposit_count", Help: "The current eth1 deposit count in the last processed state eth1data field.", }) totalEligibleBalances = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_total_eligible_balances", + Name: "total_eligible_balances", Help: "The total amount of ether, in gwei, that has been used in voting attestation target of previous epoch", }) totalVotedTargetBalances = promauto.NewGauge(prometheus.GaugeOpts{ - Name: "chain_service_total_voted_target_balances", + Name: "total_voted_target_balances", Help: "The total amount of ether, in gwei, that is eligible for voting of previous epoch", }) ) -func (s *Service) reportSlotMetrics(currentSlot uint64) { +// ReportSlotMetrics reports slot related metrics. +func ReportSlotMetrics(currentSlot uint64, headSlot uint64, headState *stateTrie.BeaconState) { beaconSlot.Set(float64(currentSlot)) - beaconHeadSlot.Set(float64(s.HeadSlot())) - if s.headState != nil { - finalizedCpt := s.headState.FinalizedCheckpoint() + beaconHeadSlot.Set(float64(headSlot)) + if headState != nil { + finalizedCpt := headState.FinalizedCheckpoint() if finalizedCpt != nil { headFinalizedEpoch.Set(float64(finalizedCpt.Epoch)) headFinalizedRoot.Set(float64(bytesutil.ToLowInt64(finalizedCpt.Root))) @@ -108,7 +94,8 @@ func (s *Service) reportSlotMetrics(currentSlot uint64) { } } -func reportEpochMetrics(state *stateTrie.BeaconState) { +// ReportEpochMetrics reports epoch related metrics. +func ReportEpochMetrics(state *stateTrie.BeaconState) { currentEpoch := state.Slot() / params.BeaconConfig().SlotsPerEpoch // Validator instances diff --git a/beacon-chain/blockchain/process_block.go b/beacon-chain/blockchain/process_block.go index 4872245639..e1b44bf196 100644 --- a/beacon-chain/blockchain/process_block.go +++ b/beacon-chain/blockchain/process_block.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-ssz" + "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/metrics" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/state" stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state" @@ -134,7 +135,7 @@ func (s *Service) onBlock(ctx context.Context, signed *ethpb.SignedBeaconBlock) // Epoch boundary bookkeeping such as logging epoch summaries. if postState.Slot() >= s.nextEpochBoundarySlot { logEpochData(postState) - reportEpochMetrics(postState) + metrics.ReportEpochMetrics(postState) // Update committees cache at epoch boundary slot. if err := helpers.UpdateCommitteeCache(postState, helpers.CurrentEpoch(postState)); err != nil { @@ -239,7 +240,7 @@ func (s *Service) onBlockInitialSyncStateTransition(ctx context.Context, signed // Epoch boundary bookkeeping such as logging epoch summaries. if postState.Slot() >= s.nextEpochBoundarySlot { - reportEpochMetrics(postState) + metrics.ReportEpochMetrics(postState) s.nextEpochBoundarySlot = helpers.StartSlot(helpers.NextEpoch(postState)) // Update committees cache at epoch boundary slot. diff --git a/beacon-chain/blockchain/receive_attestation.go b/beacon-chain/blockchain/receive_attestation.go index fdba765b9e..36b6ef0920 100644 --- a/beacon-chain/blockchain/receive_attestation.go +++ b/beacon-chain/blockchain/receive_attestation.go @@ -75,7 +75,6 @@ func (s *Service) ReceiveAttestationNoPubsub(ctx context.Context, att *ethpb.Att } } - processedAttNoPubsub.Inc() return nil } diff --git a/beacon-chain/blockchain/receive_block.go b/beacon-chain/blockchain/receive_block.go index 272af8b2ff..8ba90d65d9 100644 --- a/beacon-chain/blockchain/receive_block.go +++ b/beacon-chain/blockchain/receive_block.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-ssz" + "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/metrics" "github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute" "github.com/prysmaticlabs/prysm/beacon-chain/core/feed" statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state" @@ -56,7 +57,6 @@ func (s *Service) ReceiveBlock(ctx context.Context, block *ethpb.SignedBeaconBlo return err } - processedBlk.Inc() return nil } @@ -113,7 +113,7 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB } // Reports on block and fork choice metrics. - s.reportSlotMetrics(blockCopy.Block.Slot) + metrics.ReportSlotMetrics(blockCopy.Block.Slot, s.headSlot, s.headState) // Log state transition data. logStateTransitionData(blockCopy.Block) @@ -122,8 +122,6 @@ func (s *Service) ReceiveBlockNoPubsub(ctx context.Context, block *ethpb.SignedB defer s.epochParticipationLock.Unlock() s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances - processedBlkNoPubsub.Inc() - if featureconfig.Get().DisableForkChoice && block.Block.Slot > s.headSlot { if err := s.saveHead(ctx, blockCopy, root); err != nil { return errors.Wrap(err, "could not save head") @@ -229,7 +227,7 @@ func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *eth }) // Reports on block and fork choice metrics. - s.reportSlotMetrics(blockCopy.Block.Slot) + metrics.ReportSlotMetrics(blockCopy.Block.Slot, s.headSlot, s.headState) // Log state transition data. logStateTransitionData(blockCopy.Block) @@ -238,7 +236,6 @@ func (s *Service) ReceiveBlockNoPubsubForkchoice(ctx context.Context, block *eth defer s.epochParticipationLock.Unlock() s.epochParticipation[helpers.SlotToEpoch(blockCopy.Block.Slot)] = precompute.Balances - processedBlkNoPubsubForkchoice.Inc() return nil } @@ -304,7 +301,7 @@ func (s *Service) ReceiveBlockNoVerify(ctx context.Context, block *ethpb.SignedB }) // Reports on blockCopy and fork choice metrics. - s.reportSlotMetrics(blockCopy.Block.Slot) + metrics.ReportSlotMetrics(blockCopy.Block.Slot, s.headSlot, s.headState) // Log state transition data. log.WithFields(logrus.Fields{ @@ -329,6 +326,6 @@ func isCompetingBlock(root []byte, slot uint64, headRoot []byte, headSlot uint64 "headSlot": headSlot, "headRoot": hex.EncodeToString(headRoot), }).Warn("Calculated head diffs from new block") - competingBlks.Inc() + metrics.CompetingBlks.Inc() } }